Return $metadata resource as odata + openapi spec

Returning odata and openapi specs in json format.
Specs are scoped to the request permissions.
Different users (roles) will receive different metadata endpoints
and resources.

Change-type: minor
Signed-off-by: Harald Fischer <[email protected]>

Author: fisehara

src/odata-metadata/odata-metadata-generator.ts

@@ -0,0 +1,99 @@
+import * as odataMetadata from 'odata-openapi';
+import type { generateODataMetadata } from './odata-metadata-generator';
+import _ = require('lodash');
+// tslint:disable-next-line:no-var-requires
+
+export const generateODataMetadataAsOpenApi = (
+	odataCsdl: ReturnType<typeof generateODataMetadata>,
+	versionBasePathUrl: string = '',
+	hostname: string = '',
+) => {
+	// console.log(`odataCsdl:${JSON.stringify(odataCsdl, null, 2)}`);
+	const openAPIJson: any = odataMetadata.csdl2openapi(odataCsdl, {
+		scheme: 'https',
+		host: hostname,
+		basePath: versionBasePathUrl,
+		diagram: false,
+		maxLevels: 5,
+	});
+
+	/**
+	 * Manual rewriting OpenAPI specification to delete OData default functionality
+	 * that is not implemented in Pinejs yet or is based on PineJs implements OData V3.
+	 *
+	 * Rewrite odata body response schema properties from `value: ` to `d: `
+	 * Currently pinejs is returning `d: `
+	 * https://www.odata.org/documentation/odata-version-2-0/json-format/ (6. Representing Collections of Entries)
+	 * https://www.odata.org/documentation/odata-version-3-0/json-verbose-format/ (6.1 Response body)
+	 *
+	 * New v4 odata specifies the body response with `value: `
+	 * http://docs.oasis-open.org/odata/odata-json-format/v4.01/odata-json-format-v4.01.html#sec_IndividualPropertyorOperationRespons
+	 *
+	 *
+	 * Currently pinejs does not implement a $count=true query parameter as this would return the count of all rows returned as an additional parameter.
+	 * This was not part of OData V3 and is new for OData V4. As the odata-openapi converte is opionionated on V4 the parameter is put into the schema.
+	 * Until this is in parity with OData V4 pinejs needs to cleanup the `odata.count` key from the response schema put in by `csdl2openapi`
+	 *
+	 *
+	 * Used oasis translator generates openapi according to v4 spec (`value: `)
+	 *
+	 * Unfortunantely odata-openapi does not export the genericFilter object.
+	 * Using hardcoded generic filter description as used in odata-openapi code.
+	 * Putting the genericFilter into the #/components/parameters/filter to reference it from paths
+	 *
+	 * */
+	const parameters = openAPIJson?.components?.parameters;
+	parameters['filter'] = {
+		name: '$filter',
+		description:
+			'Filter items by property values, see [Filtering](http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#sec_SystemQueryOptionfilter)',
+		in: 'query',
+		schema: {
+			type: 'string',
+		},
+	};
+
+	for (const idx of Object.keys(openAPIJson.paths)) {
+		// rewrite `value: ` to `d: `
+		const properties =
+			openAPIJson?.paths[idx]?.get?.responses?.['200']?.content?.[
+				'application/json'
+			]?.schema?.properties;
+		if (properties?.value) {
+			properties['d'] = properties.value;
+			delete properties.value;
+		}
+
+		// cleanup the `odata.count` key from the response schema
+		if (properties?.['@odata.count']) {
+			delete properties['@odata.count'];
+		}
+
+		// copy over 'delete' and 'patch' action from single entiy path
+		// odata-openAPI converter does not support collection delete and collection update.
+		// pinejs support collection delete and update with $filter parameter
+		const entityCollectionPath = openAPIJson?.paths[idx];
+		const singleEntityPath = openAPIJson?.paths[idx + '({id})'];
+		if (entityCollectionPath != null && singleEntityPath != null) {
+			const genericFilterParameterRef = {
+				$ref: '#/components/parameters/filter',
+			};
+			for (const action of ['delete', 'patch']) {
+				entityCollectionPath[action] = _.clone(singleEntityPath?.[action]);
+				if (entityCollectionPath[action]) {
+					entityCollectionPath[action]['parameters'] = [
+						genericFilterParameterRef,
+					];
+				}
+			}
+		}
+	}
+
+	// cleanup $batch path as pinejs does not implement it.
+	// http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#sec_BatchRequests
+	if (openAPIJson?.paths['/$batch']) {
+		delete openAPIJson.paths['/$batch'];
+	}
+
+	return openAPIJson;
+};

src/odata-metadata/open-api-sepcification-generator.ts

@@ -324,7 +324,7 @@ const namespaceRelationships = (
 	});
 };
 
-type PermissionLookup = Dictionary<true | string[]>;
+export type PermissionLookup = Dictionary<true | string[]>;
 
 const getPermissionsLookup = env.createCache(
 	'permissionsLookup',
@@ -1703,7 +1703,7 @@ const getGuestPermissions = memoize(
 	{ promise: true },
 );
 
-const getReqPermissions = async (
+export const getReqPermissions = async (
 	req: PermissionReq,
 	odataBinds: ODataBinds = [] as any as ODataBinds,
 ) => {

src/sbvr-api/permissions.ts

@@ -43,7 +43,7 @@ import { ExtendedSBVRParser } from '../extended-sbvr-parser/extended-sbvr-parser
 
 import * as asyncMigrator from '../migrator/async';
 import * as syncMigrator from '../migrator/sync';
-import { generateODataMetadata } from '../odata-metadata/odata-metadata-generator';
+import { generateODataMetadataAsOpenApi } from '../odata-metadata/open-api-sepcification-generator';
 
 import type DevModel from './dev';
 // eslint-disable-next-line @typescript-eslint/no-var-requires
@@ -105,6 +105,8 @@ import {
 	type MigrationExecutionResult,
 	setExecutedMigrations,
 } from '../migrator/utils';
+import { generateODataMetadata } from '../odata-metadata/odata-metadata-generator';
+import { metadataEndpoints } from './uri-parser';
 
 const LF2AbstractSQLTranslator = LF2AbstractSQL.createTranslator(sbvrTypes);
 const LF2AbstractSQLTranslatorVersion = `${LF2AbstractSQLVersion}+${sbvrTypesVersion}`;
@@ -147,18 +149,18 @@ export interface ApiKey extends Actor {
 export interface Response {
 	statusCode: number;
 	headers?:
-		| {
-				[headerName: string]: any;
-		  }
-		| undefined;
+	| {
+		[headerName: string]: any;
+	}
+	| undefined;
 	body?: AnyObject | string;
 }
 
 export type ModelExecutionResult =
 	| undefined
 	| {
-			migrationExecutionResult?: MigrationExecutionResult;
-	  };
+		migrationExecutionResult?: MigrationExecutionResult;
+	};
 
 const memoizedResolvedSynonym = memoizeWeak(
 	(
@@ -250,9 +252,9 @@ const prettifyConstraintError = (
 		let keyMatches: RegExpExecArray | null = null;
 		let violatedConstraintInfo:
 			| {
-					table: AbstractSQLCompiler.AbstractSqlTable;
-					name: string;
-			  }
+				table: AbstractSQLCompiler.AbstractSqlTable;
+				name: string;
+			}
 			| undefined;
 		if (err instanceof db.UniqueConstraintError) {
 			switch (db.engine) {
@@ -290,8 +292,8 @@ const prettifyConstraintError = (
 				const columns = keyMatches[1].split('_');
 				throw new db.UniqueConstraintError(
 					'"' +
-						columns.map(sqlNameToODataName).join('" and "') +
-						'" must be unique.',
+					columns.map(sqlNameToODataName).join('" and "') +
+					'" must be unique.',
 				);
 			}
 			if (violatedConstraintInfo != null) {
@@ -326,16 +328,16 @@ const prettifyConstraintError = (
 					const tableName = abstractSqlModel.tables[resourceName].name;
 					keyMatches = new RegExp(
 						'"' +
-							tableName +
-							'" violates foreign key constraint ".*?" on table "(.*?)"',
+						tableName +
+						'" violates foreign key constraint ".*?" on table "(.*?)"',
 					).exec(err.message);
 					if (keyMatches == null) {
 						keyMatches = new RegExp(
 							'"' +
-								tableName +
-								'" violates foreign key constraint "' +
-								tableName +
-								'_(.*?)_fkey"',
+							tableName +
+							'" violates foreign key constraint "' +
+							tableName +
+							'_(.*?)_fkey"',
 						).exec(err.message);
 					}
 					break;
@@ -362,8 +364,8 @@ const prettifyConstraintError = (
 					case 'postgres':
 						keyMatches = new RegExp(
 							'new row for relation "' +
-								table.name +
-								'" violates check constraint "(.*?)"',
+							table.name +
+							'" violates check constraint "(.*?)"',
 						).exec(err.message);
 						break;
 				}
@@ -980,11 +982,11 @@ export const runRule = (() => {
 		const odataResult = (await runURI(
 			'GET',
 			'/' +
-				vocab +
-				'/' +
-				sqlNameToODataName(table.resourceName) +
-				'?$filter=' +
-				filter,
+			vocab +
+			'/' +
+			sqlNameToODataName(table.resourceName) +
+			'?$filter=' +
+			filter,
 			undefined,
 			undefined,
 			permissions.rootRead,
@@ -1375,7 +1377,7 @@ const runODataRequest = (req: Express.Request, vocabulary: string) => {
 							-'#canAccess'.length,
 						);
 					}
-					if (abstractSqlModel.tables[resolvedResourceName] == null) {
+					if (abstractSqlModel.tables[resolvedResourceName] == null && !metadataEndpoints.includes(resolvedResourceName)) {
 						throw new UnauthorizedError();
 					}
 
@@ -1695,19 +1697,19 @@ const runRequest = async (
 
 const runChangeSet =
 	(req: Express.Request, tx: Db.Tx) =>
-	async (
-		changeSetResults: Map<number, Response>,
-		request: uriParser.ODataRequest,
-	): Promise<void> => {
-		request = updateBinds(changeSetResults, request);
-		const result = await runRequest(req, tx, request);
-		if (request.id == null) {
-			throw new Error('No request id');
-		}
-		result.headers ??= {};
-		result.headers['content-id'] = request.id;
-		changeSetResults.set(request.id, result);
-	};
+		async (
+			changeSetResults: Map<number, Response>,
+			request: uriParser.ODataRequest,
+		): Promise<void> => {
+			request = updateBinds(changeSetResults, request);
+			const result = await runRequest(req, tx, request);
+			if (request.id == null) {
+				throw new Error('No request id');
+			}
+			result.headers ??= {};
+			result.headers['content-id'] = request.id;
+			changeSetResults.set(request.id, result);
+		};
 
 // Requests inside a changeset may refer to resources created inside the
 // changeset, the generation of the sql query for those requests must be
@@ -1878,10 +1880,35 @@ const respondGet = async (
 		return response;
 	} else {
 		if (request.resourceName === '$metadata') {
+			const permLookup = await permissions.getReqPermissions(req);
+			const spec = generateODataMetadata(
+				vocab,
+				models[vocab].abstractSql,
+				permLookup,
+			);
+			return {
+				statusCode: 200,
+				body: spec,
+				headers: { 'content-type': 'application/json' },
+			};
+		} else if (request.resourceName === 'openapi.json') {
+			// https://docs.oasis-open.org/odata/odata-openapi/v1.0/cn01/odata-openapi-v1.0-cn01.html#sec_ProvidingOASDocumentsforanODataServi
+			// Following the OASIS OData to openapi translation guide the openapi.json is an independent resource
+			const permLookup = await permissions.getReqPermissions(req);
+			const spec = generateODataMetadata(
+				vocab,
+				models[vocab].abstractSql,
+				permLookup,
+			);
+			const openApispec = generateODataMetadataAsOpenApi(
+				spec,
+				req.originalUrl.replace('openapi.json', ''),
+				req.hostname,
+			);
 			return {
 				statusCode: 200,
-				body: models[vocab].odataMetadata,
-				headers: { 'content-type': 'xml' },
+				body: openApispec,
+				headers: { 'content-type': 'application/json' },
 			};
 		} else {
 			// TODO: request.resourceName can be '$serviceroot' or a resource and we should return an odata xml document based on that

src/sbvr-api/sbvr-utils.ts

@@ -260,7 +260,7 @@ const memoizedOdata2AbstractSQL = (() => {
 	};
 })();
 
-export const metadataEndpoints = ['$metadata', '$serviceroot'];
+export const metadataEndpoints = ['$metadata', '$serviceroot', 'openapi.json'];
 
 export function parseOData(
 	b: UnparsedRequest & { _isChangeSet?: false },

src/sbvr-api/uri-parser.ts

@@ -0,0 +1,65 @@
+import { writeFileSync } from 'fs';
+import { expect } from 'chai';
+import supertest from 'supertest';
+import { testInit, testDeInit, testLocalServer } from './lib/test-init';
+
+import OpenAPIParser from '@readme/openapi-parser';
+
+describe('08 metadata / openAPI spec', function () {
+	describe('Full model access specification', async function () {
+		const fixturePath =
+			__dirname + '/fixtures/08-metadata/config-full-access.js';
+		let pineServer: Awaited<ReturnType<typeof testInit>>;
+		before(async () => {
+			pineServer = await testInit({
+				configPath: fixturePath,
+				deleteDb: true,
+			});
+		});
+
+		after(async () => {
+			await testDeInit(pineServer);
+		});
+
+		it('should send OData CSDL JSON on /$metadata', async () => {
+			const res = await supertest(testLocalServer)
+				.get('/example/$metadata')
+				.expect(200);
+			expect(res.body).to.be.an('object');
+		});
+
+		it('should send valid OpenAPI spec JSON on /$metadata', async () => {
+			const { body } = await supertest(testLocalServer)
+				.get('/example/openapi.json')
+				.expect(200);
+			expect(body).to.be.an('object');
+
+			const bodySpec = JSON.stringify(body, null, 2);
+			await writeFileSync('openApiSpe-full.json', bodySpec);
+
+			// validate the openAPI spec and expect no validator errors.
+			try {
+				const apiSpec = await OpenAPIParser.validate(JSON.parse(bodySpec));
+				expect(apiSpec).to.be.an('object');
+			} catch (err) {
+				expect(err).to.be.undefined;
+			}
+		});
+
+		it('OpenAPI spec should contain all paths and actions on resources', async () => {
+			// full CRUD access for device resource
+			const res = await supertest(testLocalServer)
+				.get('/example/openapi.json')
+				.expect(200);
+			expect(res.body).to.be.an('object');
+
+			// all collections should have get, patch, delete and post
+			const singleIdPathRegEx = /\({id}\)/;
+			for (const [path, value] of Object.entries(res.body.paths)) {
+				if (!singleIdPathRegEx.exec(path)) {
+					expect(value).to.have.keys(['get', 'patch', 'delete', 'post']);
+				}
+			}
+		});
+	});
+});