Add integrity check to webresources

Change-type: patch

Author: otaviojacobi

package.json

@@ -68,9 +68,13 @@
     "typed-error": "^3.2.2"
   },
   "devDependencies": {
+    "@aws-crypto/crc32": "^5.2.0",
+    "@aws-crypto/crc32c": "^5.2.0",
+    "@aws-sdk/crc64-nvme-crt": "^3.816.0",
+    "@aws-sdk/types": "^3.804.0",
     "@balena/lint": "^9.1.6",
     "@balena/pinejs": "file:./",
-    "@balena/pinejs-webresource-s3": "^2.1.1",
+    "@balena/pinejs-webresource-s3": "2.2.0-build-experiment-with-integrity-checks-baf4fcd6a4847aa7e8ed4123a8ac3f1c74216398-1",
     "@faker-js/faker": "^9.6.0",
     "@types/busboy": "^1.5.4",
     "@types/chai": "^5.2.1",

src/webresource-handler/actions/commitUpload.ts

@@ -11,12 +11,14 @@ import {
 import type { Response } from '../../sbvr-api/sbvr-utils.js';
 import { api } from '../../sbvr-api/sbvr-utils.js';
 import { permissions } from '../../server-glue/module.js';
+import { getChecksumParams } from '../index.js';
 import { getMultipartUploadHandler } from '../multipartUpload.js';
 
 const commitUploadAction = async ({
 	request,
 	tx,
 	id,
+	req,
 	api: applicationApi,
 }: ODataActionArgs): Promise<Response> => {
 	if (typeof id !== 'number') {
@@ -28,11 +30,14 @@ const commitUploadAction = async ({
 	const multipartUpload = await getOngoingUpload(request, id, tx);
 	const handler = getMultipartUploadHandler();
 
+	// @ts-expect-error - req.headers is there, we just need to tell them
+	const checksumPayload = getChecksumParams(req.headers);
 	const webresource = await handler.multipartUpload.commit({
 		fileKey: multipartUpload.fileKey,
 		uploadId: multipartUpload.uploadId,
 		filename: multipartUpload.filename,
 		providerCommitData: multipartUpload.providerCommitData,
+		...checksumPayload,
 	});
 
 	await Promise.all([

src/webresource-handler/index.ts

@@ -22,16 +22,40 @@ import type WebresourceModel from './webresource.js';
 import { isMultipartUploadAvailable } from './multipartUpload.js';
 import { addAction } from '../sbvr-api/actions.js';
 import { beginUpload, commitUpload, cancelUpload } from './actions/index.js';
+import type { IncomingHttpHeaders } from 'node:http';
 
 export * from './handlers/index.js';
 
-export interface IncomingFile {
+// S3 only supports full checksums (on multipart uploads) for these algorithms
+// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html#ChecksumTypes
+export const supportedChecksumAlgorithms = [
+	'CRC64NVME',
+	'CRC32',
+	'CRC32C',
+] as const;
+export type SupportedChecksumAlgorithm =
+	(typeof supportedChecksumAlgorithms)[number];
+
+type ChecksumPayload =
+	| {
+			checksum?: undefined;
+			checksumAlgorithm?: undefined;
+	  }
+	| {
+			checksum: string;
+			checksumAlgorithm: SupportedChecksumAlgorithm;
+	  };
+
+export type IncomingFile = {
 	fieldname: string;
 	originalname: string;
 	encoding: string;
 	mimetype: string;
 	stream: stream.Readable;
-}
+} & ChecksumPayload;
+
+export const checksumHeaderName = 'x-pinejs-checksum';
+export const checksumAlgorithmHeaderName = 'x-pinejs-checksum-algorithm';
 
 export interface UploadResponse {
 	size: number;
@@ -57,12 +81,12 @@ export interface BeginMultipartUploadHandlerResponse {
 	uploadId: string;
 }
 
-export interface CommitMultipartUploadPayload {
+export type CommitMultipartUploadPayload = {
 	fileKey: string;
 	uploadId: string;
 	filename: string;
 	providerCommitData?: Record<string, any>;
-}
+} & ChecksumPayload;
 
 export interface CancelMultipartUploadPayload {
 	fileKey: string;
@@ -159,6 +183,31 @@ const getRequestUploadValidator = async (
 	};
 };
 
+export const getChecksumParams = (headers: IncomingHttpHeaders) => {
+	const checksum = headers[checksumHeaderName];
+	const checksumAlgorithm = headers[checksumAlgorithmHeaderName] as
+		| SupportedChecksumAlgorithm
+		| undefined;
+
+	if (checksum == null || checksumAlgorithm == null) {
+		return { checksum: undefined, checksumAlgorithm: undefined };
+	}
+
+	if (typeof checksum !== 'string' || typeof checksumAlgorithm !== 'string') {
+		throw new errors.BadRequestError(
+			`Invalid ${checksumHeaderName} or ${checksumAlgorithmHeaderName} header`,
+		);
+	}
+
+	if (!supportedChecksumAlgorithms.includes(checksumAlgorithm)) {
+		throw new errors.BadRequestError(
+			`Invalid ${checksumAlgorithmHeaderName} header value: ${checksumAlgorithm}`,
+		);
+	}
+
+	return { checksum, checksumAlgorithm };
+};
+
 export const getUploaderMiddlware = (
 	handler: WebResourceHandler,
 ): Express.RequestHandler => {
@@ -218,14 +267,17 @@ export const getUploaderMiddlware = (
 							filestream.resume();
 							return;
 						}
+						const checksumPayload = getChecksumParams(req.headers);
 						const file: IncomingFile = {
 							originalname: info.filename,
 							encoding: info.encoding,
 							mimetype: info.mimeType,
 							stream: filestream,
 							fieldname,
+							...checksumPayload,
 						};
 						const result = await handler.handleFile(file);
+
 						req.body[fieldname] = {
 							filename: info.filename,
 							content_type: info.mimeType,

test/06-webresource.test.ts

@@ -22,7 +22,16 @@ import {
 import { intVar, requiredVar } from '@balena/env-parsing';
 import { assertExists } from './lib/common.js';
 import { PineTest } from 'pinejs-client-supertest';
-import type { UploadPart } from '../out/webresource-handler/index.js';
+import {
+	type SupportedChecksumAlgorithm,
+	supportedChecksumAlgorithms,
+	type UploadPart,
+} from '../out/webresource-handler/index.js';
+import { CrtCrc64Nvme } from '@aws-sdk/crc64-nvme-crt';
+import { AwsCrc32 } from '@aws-crypto/crc32';
+import { AwsCrc32c } from '@aws-crypto/crc32c';
+import type { Checksum } from '@aws-sdk/types';
+import { setTimeout } from 'timers/promises';
 
 const pipeline = util.promisify(pipelineRaw);
 
@@ -122,6 +131,41 @@ describe('06 webresources tests', function () {
 					);
 				});
 
+				supportedChecksumAlgorithms.forEach((checksumAlgorithm) => {
+					it(`accepts a checksum ${checksumAlgorithm} header ${resourcePath}`, async () => {
+						const { body: organization } = await supertest(testLocalServer)
+							.post(`/${resourceName}/organization`)
+							.set('x-pinejs-checksum-algorithm', checksumAlgorithm)
+							.set(
+								'x-pinejs-checksum',
+								await getChecksum(checksumAlgorithm, filePath),
+							)
+							.field('name', 'John')
+							.attach(resourcePath, filePath, { filename, contentType })
+							.expect(201);
+
+						expect(organization[resourcePath].size).to.equals(fileSize);
+						expect(organization[resourcePath].filename).to.equals(filename);
+						expect(organization[resourcePath].content_type).to.equals(
+							contentType,
+						);
+					});
+				});
+
+				supportedChecksumAlgorithms.forEach((checksumAlgorithm) => {
+					it(`fails if checksum ${checksumAlgorithm} is wrong`, async () => {
+						await supertest(testLocalServer)
+							.post(`/${resourceName}/organization`)
+							.set('x-pinejs-checksum-algorithm', checksumAlgorithm)
+							.set('x-pinejs-checksum', 'abc')
+							.field('name', 'John')
+							.attach(resourcePath, filePath, { filename, contentType })
+							.expect(400);
+
+						expect(await isBucketEventuallyEmpty()).to.be.true;
+					});
+				});
+
 				it(`does not store ${resourcePath} if is bigger than PINEJS_WEBRESOURCE_MAXFILESIZE`, async () => {
 					const { largeStream } = await getLargeFileStream(
 						intVar('PINEJS_WEBRESOURCE_MAXFILESIZE') + 10 * 1024 * 1024,
@@ -1405,14 +1449,12 @@ const getKeyFromHref = (href: string): string => {
 	return splittedHref[splittedHref.length - 1];
 };
 
-const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-
 const isBucketEventuallyEmpty = async (attempts = 5, retryDelay = 1000) => {
 	for (let attempt = 0; attempt < attempts; attempt++) {
 		if ((await listAllFilesInBucket()).length === 0) {
 			return true;
 		}
-		await delay(retryDelay);
+		await setTimeout(retryDelay);
 	}
 
 	return false;
@@ -1430,7 +1472,7 @@ const isEventuallyDeleted = async (
 		if (!fileExists) {
 			return true;
 		}
-		await delay(retryDelay);
+		await setTimeout(retryDelay);
 	}
 
 	return false;
@@ -1557,8 +1599,8 @@ const awaitForDeletionTasks = async (
 	initialDelay = 500,
 ) => {
 	// Even the creation of deletion tasks happens in second plan
-	// so we give it a initial delay for the tasks to be created
-	await delay(initialDelay);
+	// so we give it a initial setTimeout for the tasks to be created
+	await setTimeout(initialDelay);
 	for (let i = 0; i < attempts; i++) {
 		const { body: pendingDeletions } = await pineTask.get({
 			resource: 'task',
@@ -1575,7 +1617,7 @@ const awaitForDeletionTasks = async (
 			return;
 		}
 
-		await delay(retryDelay);
+		await setTimeout(retryDelay);
 	}
 
 	throw new Error('Failed to await for webresource deletions');
@@ -1650,3 +1692,23 @@ const uploadParts = async (parts: UploadPart[]) => {
 		})),
 	};
 };
+
+const getChecksum = async (
+	algorithm: SupportedChecksumAlgorithm,
+	filePath: string,
+): Promise<string> => {
+	const fileBuffer = await fs.readFile(filePath);
+	let calculate: Checksum;
+	if (algorithm === 'CRC32') {
+		calculate = new AwsCrc32();
+	} else if (algorithm === 'CRC32C') {
+		calculate = new AwsCrc32c();
+	} else if (algorithm === 'CRC64NVME') {
+		calculate = new CrtCrc64Nvme();
+	} else {
+		throw new Error(`Unsupported algorithm: ${algorithm}`);
+	}
+
+	calculate.update(fileBuffer);
+	return Buffer.from(await calculate.digest()).toString('base64');
+};