Allow multipart uploads of large files

Change-type: minor

Author: otaviojacobi

package.json

@@ -20,7 +20,7 @@
     "webpack-build": "npm run webpack-browser && npm run webpack-module && npm run webpack-server",
     "lint": "balena-lint -t tsconfig.dev.json -e js -e ts src test build typings Gruntfile.cts && npx tsc --project tsconfig.dev.json --noEmit",
     "test": "npm run build && npm run lint && npm run webpack-build && npm run test:compose && npm run test:generated-types",
-    "test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' INT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 PINEJS_QUEUE_CONCURRENCY=1 TZ=UTC npx mocha",
+    "test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' INT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 PINEJS_QUEUE_CONCURRENCY=1 TZ=UTC PINEJS_WEBRESOURCE_MULTIPART_ENABLED=true npx mocha",
     "test:generated-types": "npm run generate-types && git diff --exit-code ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts",
     "lint-fix": "balena-lint -t tsconfig.dev.json -e js -e ts --fix src test build typings Gruntfile.cts",
     "generate-types": "node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/user.sbvr ./src/sbvr-api/user.ts && node ./bin/sbvr-compiler.js generate-types ./src/migrator/migrations.sbvr ./src/migrator/migrations.ts && node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/dev.sbvr ./src/sbvr-api/dev.ts && node ./bin/sbvr-compiler.js generate-types ./src/tasks/tasks.sbvr ./src/tasks/tasks.ts && node ./bin/sbvr-compiler.js generate-types ./src/webresource-handler/webresource.sbvr ./src/webresource-handler/webresource.ts && balena-lint -t tsconfig.dev.json --fix ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts"
@@ -70,7 +70,7 @@
   "devDependencies": {
     "@balena/lint": "^9.1.6",
     "@balena/pinejs": "file:./",
-    "@balena/pinejs-webresource-s3": "^1.0.4",
+    "@balena/pinejs-webresource-s3": "2.0.0-build-new-multiparthandle-interface-66b03581234eaa7ce15c6f389e39b5b7ed3d1bc5-1",
     "@faker-js/faker": "^9.6.0",
     "@types/busboy": "^1.5.4",
     "@types/chai": "^5.2.1",
@@ -145,7 +145,7 @@
     ],
     "loader": "ts-node/esm/transpile-only",
     "exit": true,
-    "timeout": 60000,
+    "timeout": 6000000,
     "recursive": true
   },
   "versionist": {

src/config-loader/env.ts

@@ -159,6 +159,17 @@ export const tasks = {
 	queueIntervalMS: intVar('PINEJS_QUEUE_INTERVAL_MS', 1000),
 };
 
+export const webResource = {
+	multipartUploadEnabled: boolVar(
+		'PINEJS_WEBRESOURCE_MULTIPART_ENABLED',
+		false,
+	),
+	singleUploadMaxFilesize: intVar(
+		'PINEJS_WEBRESOURCE_MAXFILESIZE',
+		299 * 1024 * 1024,
+	),
+};
+
 export const guardTestMockOnly = () => {
 	if (process.env.DEPLOYMENT !== 'TEST') {
 		throw new Error('Attempting to use TEST_MOCK_ONLY outside of tests');

src/sbvr-api/sbvr-utils.ts

@@ -1140,7 +1140,7 @@ const getFinalAbstractSqlModel = (
 	return (request.finalAbstractSqlModel ??= models[finalModel].abstractSql);
 };
 
-const getIdField = (
+export const getIdField = (
 	request: Pick<
 		uriParser.ODataRequest,
 		| 'translateVersions'

src/webresource-handler/index.ts

@@ -18,8 +18,13 @@ import { TypedError } from 'typed-error';
 import type { Resolvable } from '../sbvr-api/common-types.js';
 import type WebresourceModel from './webresource.js';
 import { importSBVR } from '../server-glue/sbvr-loader.js';
+import {
+	isMultipartUploadAvailable,
+	multipartUploadHooks,
+} from './multipartUpload.js';
 
 export * from './handlers/index.js';
+export type { BeginUploadResponse } from './multipartUpload.js';
 
 export interface IncomingFile {
 	fieldname: string;
@@ -332,6 +337,10 @@ const throwIfWebresourceNotInMultipart = (
 	{ req, request }: HookArgs,
 ) => {
 	if (
+		request.custom.isAction !== 'beginUpload' &&
+		request.custom.isAction !== 'commitUpload' &&
+		request.custom.isAction !== 'cancelUpload' &&
+		req.user !== permissions.root.user &&
 		!req.is?.('multipart') &&
 		webResourceFields.some((field) => request.values[field] != null)
 	) {
@@ -528,6 +537,15 @@ export const setupUploadHooks = (
 		resourceName,
 		getCreateWebResourceHooks(handler),
 	);
+
+	if (isMultipartUploadAvailable(handler)) {
+		sbvrUtils.addPureHook(
+			'POST',
+			apiRoot,
+			resourceName,
+			multipartUploadHooks(handler),
+		);
+	}
 };
 
 const initSql = `