diff --git a/.gitignore b/.gitignore index 2485740..de95899 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,4 @@ node_modules/ test-params.env extensions/test-params.env /typesense-server-data/ +typesense-data diff --git a/docker-compose.yml b/docker-compose.yml index 560bd9c..06cfd5a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: typesense: - image: typesense/typesense:27.0 + image: typesense/typesense:29.0.rc13 restart: on-failure ports: - "8108:8108" diff --git a/extension.yaml b/extension.yaml index 42b5b46..e166ef4 100644 --- a/extension.yaml +++ b/extension.yaml @@ -349,3 +349,48 @@ params: value: true default: false required: false + - param: TYPESENSE_USE_BUFFER + label: Use Buffer for Typesense Operations + description: >- + Enable buffering for Typesense operations. Buffering can improve performance by batching operations + and providing retry capabilities for failed requests. + type: select + options: + - label: No + value: false + - label: Yes + value: true + default: false + required: false + - param: TYPESENSE_BUFFER_COLLECTION_IN_FIRESTORE + label: Typesense Buffer Collection in Firestore + description: >- + The Firestore collection name to use for buffering Typesense operations. + Only applicable if buffering is enabled. + example: typesense_buffer + default: typesense_buffer + required: false + - param: TYPESENSE_BUFFER_BATCH_SIZE + label: Typesense Buffer Batch Size + description: >- + The number of documents to process in a single batch when flushing the buffer. + Only applicable if buffering is enabled. + example: 100 + default: 100 + required: false + - param: TYPESENSE_BUFFER_MAX_RETRIES + label: Typesense Buffer Maximum Retries + description: >- + The maximum number of retry attempts for failed Typesense operations. + Only applicable if buffering is enabled. + example: 3 + default: 3 + required: false + - param: TYPESENSE_BUFFER_FLUSH_INTERVAL + label: Typesense Buffer Flush Interval + description: >- + The time interval in cron-style format at which the buffer should be automatically flushed. + Default is "every 3 minutes". Syntax is documented [here](https://cloud.google.com/appengine/docs/flexible/scheduling-jobs-with-cron-yaml#formatting_the_schedule). [Unix crontab](http://man7.org/linux/man-pages/man5/crontab.5.html) is also supported. Only applicable if buffering is enabled. + example: every 5 minutes + default: every 3 minutes + required: false diff --git a/extensions/test-params-buffer-enabled.local.env b/extensions/test-params-buffer-enabled.local.env new file mode 100644 index 0000000..0eb85a0 --- /dev/null +++ b/extensions/test-params-buffer-enabled.local.env @@ -0,0 +1,11 @@ +LOCATION=us-central1 +FIRESTORE_DATABASE_REGION=nam5 +FIRESTORE_COLLECTION_PATH=books +FIRESTORE_COLLECTION_FIELDS=author,title,rating,isAvailable,location,createdAt,nested_field,tags,nullField,ref +FLATTEN_NESTED_DOCUMENTS=false +TYPESENSE_HOSTS=localhost +TYPESENSE_PORT=8108 +TYPESENSE_PROTOCOL=http +TYPESENSE_COLLECTION_NAME=books_firestore/1 +TYPESENSE_API_KEY=xyz +TYPESENSE_USE_BUFFER=true diff --git a/extensions/test-params-subcategory-buffer-enabled.local.env b/extensions/test-params-subcategory-buffer-enabled.local.env new file mode 100644 index 0000000..14a4b9d --- /dev/null +++ b/extensions/test-params-subcategory-buffer-enabled.local.env @@ -0,0 +1,14 @@ +LOCATION=us-central1 +FIRESTORE_DATABASE_REGION=nam5 +FIRESTORE_COLLECTION_PATH=users/{personId}/books +TEST_FIRESTORE_PARENT_COLLECTION_PATH=users +TEST_FIRESTORE_PARENT_ID=123 +TEST_FIRESTORE_CHILD_FIELD_NAME=books +FIRESTORE_COLLECTION_FIELDS=author,title,rating,isAvailable,location,createdAt,nested_field,tags,nullField,ref +FLATTEN_NESTED_DOCUMENTS=false +TYPESENSE_HOSTS=localhost +TYPESENSE_PORT=8108 +TYPESENSE_PROTOCOL=http +TYPESENSE_COLLECTION_NAME=books_firestore/1 +TYPESENSE_API_KEY=xyz +TYPESENSE_USE_BUFFER=true \ No newline at end of file diff --git a/functions/package-lock.json b/functions/package-lock.json index 603c741..777fd93 100644 --- a/functions/package-lock.json +++ b/functions/package-lock.json @@ -8,10 +8,10 @@ "dependencies": { "@babel/runtime": "^7.24.7", "dotenv": "^16.4.7", - "firebase-admin": "^13.0.2", - "firebase-functions": "^6.2.0", + "firebase-admin": "^13.2.0", + "firebase-functions": "^6.3.2", "lodash.get": "^4.4.2", - "typesense": "^1.8.2" + "typesense": "^2.1.0-3" }, "devDependencies": { "eslint": "^8.52.0", @@ -716,9 +716,10 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/axios": { - "version": "1.7.7", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", - "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.4.tgz", + "integrity": "sha512-eBSYY4Y68NNlHbHBMdeDmKNtDgXWhQsJcGqzO3iLUM0GraQFSS9cVgPX5I9b3lbdFKyYoAEGAZF1DwhTaljNAw==", + "license": "MIT", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -726,12 +727,14 @@ } }, "node_modules/axios/node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", + "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", "mime-types": "^2.1.12" }, "engines": { @@ -848,6 +851,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -1090,6 +1106,20 @@ "url": "https://dotenvx.com" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/duplexify": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", @@ -1139,12 +1169,10 @@ } }, "node_modules/es-define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", - "dependencies": { - "get-intrinsic": "^1.2.4" - }, + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", "engines": { "node": ">= 0.4" } @@ -1157,6 +1185,33 @@ "node": ">= 0.4" } }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -1545,9 +1600,10 @@ } }, "node_modules/firebase-admin": { - "version": "13.0.2", - "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-13.0.2.tgz", - "integrity": "sha512-YWVpoN+tZVSRXF0qC0gojoF5bSqvBRbnBk8+xUtFiguM2L4vB7f0moAwV1VVWDDHvTnvQ68OyTMpdp6wKo/clw==", + "version": "13.2.0", + "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-13.2.0.tgz", + "integrity": "sha512-qQBTKo0QWCDaWwISry989pr8YfZSSk00rNCKaucjOgltEm3cCYzEe4rODqBd1uUwma+Iu5jtAzg89Nfsjr3fGg==", + "license": "Apache-2.0", "dependencies": { "@fastify/busboy": "^3.0.0", "@firebase/database-compat": "^2.0.0", @@ -1569,9 +1625,10 @@ } }, "node_modules/firebase-functions": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/firebase-functions/-/firebase-functions-6.2.0.tgz", - "integrity": "sha512-vfyyVHS8elxplzEQ9To+NaINRPFUsDasQrasTa2eFJBYSPzdhkw6rwLmvwyYw622+ze+g4sDIb14VZym+afqXQ==", + "version": "6.3.2", + "resolved": "https://registry.npmjs.org/firebase-functions/-/firebase-functions-6.3.2.tgz", + "integrity": "sha512-FC3A1/nhqt1ZzxRnj5HZLScQaozAcFSD/vSR8khqSoFNOfxuXgwJS6ZABTB7+v+iMD5z6Mmxw6OfqITUBuI7OQ==", + "license": "MIT", "dependencies": { "@types/cors": "^2.8.5", "@types/express": "^4.17.21", @@ -1609,15 +1666,16 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.15.6", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", - "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", "funding": [ { "type": "individual", "url": "https://github.com/sponsors/RubenVerborgh" } ], + "license": "MIT", "engines": { "node": ">=4.0" }, @@ -1727,15 +1785,21 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -1744,6 +1808,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/glob": { "version": "7.1.7", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", @@ -1844,11 +1921,12 @@ } }, "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dependencies": { - "get-intrinsic": "^1.1.3" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -1883,10 +1961,11 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/has-proto": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", - "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -1894,10 +1973,14 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, "engines": { "node": ">= 0.4" }, @@ -2370,6 +2453,15 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -2690,7 +2782,8 @@ "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" }, "node_modules/pseudomap": { "version": "1.0.2", @@ -3238,12 +3331,14 @@ } }, "node_modules/typesense": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/typesense/-/typesense-1.8.2.tgz", - "integrity": "sha512-aBpePjA99Qvo+OP2pJwMpvga4Jrm1Y2oV5NsrWXBxlqUDNEUCPZBIksPv2Hq0jxQxHhLLyJVbjXjByXsvpCDVA==", - "dependencies": { - "axios": "^1.6.0", - "loglevel": "^1.8.1" + "version": "2.1.0-3", + "resolved": "https://registry.npmjs.org/typesense/-/typesense-2.1.0-3.tgz", + "integrity": "sha512-/RHgarvw0CZVE+e6bG9N37KeX2zjJC4nojwbsIKhRzGt7FKNakgwJXDh1CdSp2QjlMlqf75z8YBa9ffNYThbdg==", + "license": "Apache-2.0", + "dependencies": { + "axios": "^1.8.4", + "loglevel": "^1.8.1", + "tslib": "^2.6.2" }, "engines": { "node": ">=18" diff --git a/functions/package.json b/functions/package.json index 1d2d3a7..16d5801 100644 --- a/functions/package.json +++ b/functions/package.json @@ -17,10 +17,10 @@ "dependencies": { "@babel/runtime": "^7.24.7", "dotenv": "^16.4.7", - "firebase-admin": "^13.0.2", - "firebase-functions": "^6.2.0", + "firebase-admin": "^13.2.0", + "firebase-functions": "^6.3.2", "lodash.get": "^4.4.2", - "typesense": "^1.8.2" + "typesense": "^2.1.0-3" }, "devDependencies": { "eslint": "^8.52.0", diff --git a/functions/src/backfill.js b/functions/src/backfill.js index 53b0840..a780768 100644 --- a/functions/src/backfill.js +++ b/functions/src/backfill.js @@ -90,7 +90,7 @@ module.exports = onDocumentWritten("typesense_sync/backfill", async (snapshot, c lastDoc = thisBatch.docs.at(-1) ?? null; try { - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().import(currentDocumentsBatch, {action: "upsert", return_id: true}); + await typesense.collections(config.typesenseCollectionName).documents().import(currentDocumentsBatch, {action: "upsert", return_id: true}); info(`Imported ${currentDocumentsBatch.length} documents into Typesense`); } catch (err) { error(`Import error in a batch of documents from ${currentDocumentsBatch[0].id} to ${lastDoc.id}`, err); diff --git a/functions/src/config.js b/functions/src/config.js index 71a7b5f..47a83a3 100644 --- a/functions/src/config.js +++ b/functions/src/config.js @@ -12,5 +12,10 @@ module.exports = { typesenseCollectionName: process.env.TYPESENSE_COLLECTION_NAME, typesenseAPIKey: process.env.TYPESENSE_API_KEY, typesenseBackfillTriggerDocumentInFirestore: "typesense_sync/backfill", - typesenseBackfillBatchSize: 1000, + typesenseBackfillBatchSize: process.env.TYPESENSE_BACKFILL_BATCH_SIZE || 1000, + typesenseUseBuffer: process.env.TYPESENSE_USE_BUFFER === "true", + typesenseBufferCollectionInFirestore: process.env.TYPESENSE_BUFFER_COLLECTION_IN_FIRESTORE || "typesense_buffer", + typesenseBufferBatchSize: process.env.TYPESENSE_BUFFER_BATCH_SIZE || 100, + typesenseBufferMaxRetries: process.env.TYPESENSE_BUFFER_MAX_RETRIES || 3, + typesenseBufferFlushInterval: process.env.TYPESENSE_BUFFER_FLUSH_INTERVAL || "every 3 minutes", }; diff --git a/functions/src/indexOnWrite.js b/functions/src/indexOnWrite.js index c3fded7..e87fb4a 100644 --- a/functions/src/indexOnWrite.js +++ b/functions/src/indexOnWrite.js @@ -3,15 +3,58 @@ const config = require("./config.js"); const utils = require("./utils.js"); const createTypesenseClient = require("./createTypesenseClient.js"); const {onDocumentWritten} = require("firebase-functions/v2/firestore"); +const admin = require("firebase-admin"); exports.indexOnWrite = onDocumentWritten(`${config.firestoreCollectionPath}/{docId}`, async (snapshot, _) => { const typesense = createTypesenseClient(); + if (config.typesenseUseBuffer) { + return await bufferedWrites(snapshot); + } + + return await realTimeWrites(snapshot, typesense); +}); + +const bufferedWrites = async (snapshot) => { + if (snapshot.data.after.data() == null) { + // Delete + const documentId = snapshot.data.before.id; + debug(`Buffering delete for document ${documentId}`); + + return await admin.firestore().collection(config.typesenseBufferCollectionInFirestore).add({ + documentId: documentId, + type: "delete", + status: "pending", + timestamp: Date.now(), + retries: 0, + pathParams: snapshot.params, + }); + } else { + // Create / update + const latestSnapshot = await snapshot.data.after.ref.get(); + const document = latestSnapshot.data(); + const documentId = latestSnapshot.id; + + debug(`Buffering upsert for document ${documentId}`); + + return await admin.firestore().collection(config.typesenseBufferCollectionInFirestore).add({ + documentId: documentId, + document: document, + type: "upsert", + status: "pending", + timestamp: Date.now(), + retries: 0, + pathParams: snapshot.params, + }); + } +}; + +const realTimeWrites = async (snapshot, typesense) => { if (snapshot.data.after.data() == null) { // Delete const documentId = snapshot.data.before.id; debug(`Deleting document ${documentId}`); - return await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents(encodeURIComponent(documentId)).delete(); + return await typesense.collections(config.typesenseCollectionName).documents(documentId).delete(); } else { // Create / update @@ -24,6 +67,6 @@ exports.indexOnWrite = onDocumentWritten(`${config.firestoreCollectionPath}/{doc } else { debug(`Upserting document ${typesenseDocument.id}`); } - return await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().upsert(typesenseDocument); + return await typesense.collections(config.typesenseCollectionName).documents().upsert(typesenseDocument); } -}); +}; diff --git a/functions/src/processBuffer.js b/functions/src/processBuffer.js new file mode 100644 index 0000000..e1b3987 --- /dev/null +++ b/functions/src/processBuffer.js @@ -0,0 +1,253 @@ +const {onSchedule} = require("firebase-functions/v2/scheduler"); +const {info, debug, error} = require("firebase-functions/logger"); +const config = require("./config.js"); +const createTypesenseClient = require("./createTypesenseClient.js"); +const {typesenseDocumentFromSnapshot} = require("./utils.js"); + +const admin = require("firebase-admin"); +const {default: ImportError} = require("typesense/lib/Typesense/Errors/ImportError.js"); + +admin.initializeApp({ + credential: admin.credential.applicationDefault(), +}); + +exports.processBuffer = onSchedule(config.typesenseBufferFlushInterval, async (event) => { + await processTypesenseBuffer(); +}); + +const createDeleteFilter = (ids) => { + const commaSeparatedIds = ids.join(","); + return `id:[${commaSeparatedIds}]`; +}; + +/** + * Fetches pending buffer documents from Firestore + * @return {Promise} Query snapshot of buffer documents + */ +const fetchPendingBufferDocuments = async () => { + const bufferRef = admin + .firestore() + .collection(config.typesenseBufferCollectionInFirestore) + .where("status", "in", ["pending", "retrying"]) + .where("retries", "<=", config.typesenseBufferMaxRetries) + .orderBy("timestamp") + .limit(config.typesenseBufferBatchSize); + + return bufferRef.get(); +}; + +/** + * Organizes buffer documents into batches for processing + * @param {FirebaseFirestore.QuerySnapshot} bufferDocs - Query snapshot of buffer documents + * @return {Object} Object containing categorized batches and document references + */ +const organizeBatchOperations = async (bufferDocs) => { + const upsertBatch = []; + const deleteBatch = []; + const processBatch = admin.firestore().batch(); + const docRefs = new Map(); + + bufferDocs.forEach((doc) => { + const data = doc.data(); + const documentId = data.documentId; + + processBatch.update(doc.ref, {status: "processing"}); + docRefs.set(documentId, doc.ref); + + if (data.type === "upsert") { + upsertBatch.push({ + id: documentId, + ...data.document, + pathParams: data.pathParams, + }); + } else if (data.type === "delete") { + deleteBatch.push(documentId); + } + }); + + await processBatch.commit(); + + return { + upsertBatch, + deleteBatch, + docRefs, + }; +}; + +/** + * Marks documents as completed in the buffer + * @param {string[]} documentIds - IDs of documents to mark as completed + * @param {Map} docRefs - Map of document IDs to Firestore references + * @return {Promise} + */ +const markDocumentsAsCompleted = async (documentIds, docRefs) => { + const completionBatch = admin.firestore().batch(); + + documentIds.forEach((documentId) => { + const ref = docRefs.get(documentId); + if (ref) { + completionBatch.update(ref, {status: "completed"}); + } + }); + + await completionBatch.commit(); +}; + +/** + * Updates document status based on processing result + * @param {FirebaseFirestore.DocumentReference} docRef - Firestore reference to the document + * @param {string} errorMessage - Error message to store + * @param {FirebaseFirestore.WriteBatch} batch - Firestore write batch to add the operation to + * @return {Promise} + */ +const updateDocumentStatus = async (docRef, errorMessage, batch) => { + const doc = await docRef.get(); + const data = doc.data(); + + if (data.retries >= config.typesenseBufferMaxRetries) { + batch.update(docRef, { + status: "failed", + lastError: errorMessage, + }); + } else { + batch.update(docRef, { + status: "retrying", + retries: data.retries + 1, + lastError: errorMessage, + }); + } +}; + +/** + * Processes upsert operations in Typesense + * @param {Array} upsertBatch - Array of documents to upsert + * @param {Map} docRefs - Map of document IDs to Firestore references + * @param {Object} typesense - Typesense client instance + * @return {Promise} + */ +const processUpsertOperations = async (upsertBatch, docRefs, typesense) => { + if (upsertBatch.length === 0) return; + + debug(`Upserting ${upsertBatch.length} documents`); + + try { + const typesenseDocuments = await Promise.all( + upsertBatch.map(async (doc) => { + const {pathParams, ...document} = doc; + return await typesenseDocumentFromSnapshot({id: doc.id, data: () => document}, pathParams); + }), + ); + + await typesense.collections(config.typesenseCollectionName).documents().import(typesenseDocuments, {action: "upsert", return_id: true}); + + debug(`Successfully upserted ${typesenseDocuments.length} documents`); + + // Mark all documents as completed + const documentIds = typesenseDocuments.map((doc) => doc.id); + await markDocumentsAsCompleted(documentIds, docRefs); + } catch (err) { + const isImportError = err instanceof ImportError; + const errorMessage = isImportError ? `Error upserting documents: ${JSON.stringify(err.payload.failedItems)}` : `Error upserting documents: ${err.message}`; + + error(errorMessage); + + const completionBatch = admin.firestore().batch(); + + if (isImportError && err.payload && err.payload.failedItems) { + const failedIds = new Map(); + err.payload.failedItems.forEach((item) => failedIds.set(item.id, {error: item.error})); + + for (const [documentId, docRef] of docRefs.entries()) { + if (failedIds.has(documentId)) { + const lastError = failedIds.get(documentId).error ?? err.message ?? "Unknown error"; + await updateDocumentStatus(docRef, lastError, completionBatch); + } + } + } + + await completionBatch.commit(); + } +}; + +/** + * Processes delete operations in Typesense + * @param {string[]} deleteBatch - Array of document IDs to delete + * @param {Map} docRefs - Map of document IDs to Firestore references + * @param {Object} typesense - Typesense client instance + * @return {Promise} + */ +const processDeleteOperations = async (deleteBatch, docRefs, typesense) => { + if (deleteBatch.length === 0) return; + + debug(`Deleting ${deleteBatch.length} documents`); + + try { + const result = await typesense + .collections(config.typesenseCollectionName) + .documents() + .delete({ + filter_by: createDeleteFilter(deleteBatch), + batch_size: config.typesenseBufferBatchSize, + return_id: true, + }); + + if (result.num_deleted !== deleteBatch.length) { + const completionBatch = admin.firestore().batch(); + const missing = result.num_deleted === 0 ? deleteBatch : deleteBatch.filter((id) => !result.ids.includes(id)); + + for (const id of missing) { + const docRef = docRefs.get(id); + if (docRef) { + await updateDocumentStatus(docRef, "Missing from Typesense", completionBatch); + } + } + + await completionBatch.commit(); + error(`Missing ${missing.length} documents from delete batch: ${missing.join(", ")}`); + } else { + debug(`Successfully deleted ${deleteBatch.length} documents`); + await markDocumentsAsCompleted(deleteBatch, docRefs); + } + } catch (err) { + error(`Error deleting documents: ${err.message}`); + + const completionBatch = admin.firestore().batch(); + + for (const documentId of deleteBatch) { + const docRef = docRefs.get(documentId); + if (docRef) { + await updateDocumentStatus(docRef, err.message, completionBatch); + } + } + + await completionBatch.commit(); + } +}; + +/** + * Main function to process the Typesense buffer + * @return {Promise} + */ +const processTypesenseBuffer = async () => { + const typesense = createTypesenseClient(); + + info("Processing buffer"); + + const bufferDocs = await fetchPendingBufferDocuments(); + + if (bufferDocs.empty) { + info("No documents to process"); + return; + } + + info(`Processing ${bufferDocs.size} documents`); + + const {upsertBatch, deleteBatch, docRefs} = await organizeBatchOperations(bufferDocs); + + await processUpsertOperations(upsertBatch, docRefs, typesense); + await processDeleteOperations(deleteBatch, docRefs, typesense); + + info(`Completed processing ${bufferDocs.size} documents`); +}; + +exports.processTypesenseBuffer = processTypesenseBuffer; diff --git a/package-lock.json b/package-lock.json index 8c4efa7..ba99e5f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,13 +12,13 @@ "eslint-config-google": "^0.14.0", "eslint-config-prettier": "^10.0.1", "firebase-admin": "^13.0.2", - "firebase-functions": "^6.2.0", + "firebase-functions": "^6.3.2", "firebase-functions-test": "^3.4.0", "jest": "^29.7.0", "jest-dev-server": "^10.1.0", "jest-junit": "^16.0.0", "prettier": "^3.4.2", - "typesense": "^1.8.2" + "typesense": "^2.1.0-3" }, "engines": { "node": "22" @@ -1941,7 +1941,9 @@ "license": "MIT" }, "node_modules/axios": { - "version": "1.7.7", + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.4.tgz", + "integrity": "sha512-eBSYY4Y68NNlHbHBMdeDmKNtDgXWhQsJcGqzO3iLUM0GraQFSS9cVgPX5I9b3lbdFKyYoAEGAZF1DwhTaljNAw==", "dev": true, "license": "MIT", "dependencies": { @@ -3268,9 +3270,9 @@ } }, "node_modules/firebase-admin": { - "version": "13.1.0", - "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-13.1.0.tgz", - "integrity": "sha512-XPKiTyPyvUMZ22EPk4M1oSiZ8/4qFeYwjK88o/DYpGtNbOLKrM6Oc9jTaK+P6Vwn3Vr1+OCyLLJ93Bci382UqA==", + "version": "13.2.0", + "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-13.2.0.tgz", + "integrity": "sha512-qQBTKo0QWCDaWwISry989pr8YfZSSk00rNCKaucjOgltEm3cCYzEe4rODqBd1uUwma+Iu5jtAzg89Nfsjr3fGg==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -6543,12 +6545,15 @@ } }, "node_modules/typesense": { - "version": "1.8.2", + "version": "2.1.0-3", + "resolved": "https://registry.npmjs.org/typesense/-/typesense-2.1.0-3.tgz", + "integrity": "sha512-/RHgarvw0CZVE+e6bG9N37KeX2zjJC4nojwbsIKhRzGt7FKNakgwJXDh1CdSp2QjlMlqf75z8YBa9ffNYThbdg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "axios": "^1.6.0", - "loglevel": "^1.8.1" + "axios": "^1.8.4", + "loglevel": "^1.8.1", + "tslib": "^2.6.2" }, "engines": { "node": ">=18" diff --git a/package.json b/package.json index ee23fe8..2bf8061 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,10 @@ "scripts": { "emulator": "cross-env DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:start --import=emulator_data", "export": "firebase emulators:export emulator_data", - "test": "npm run test:flatttened && npm run test:unflattened && npm run test:subcollection", + "test": "npm run test:flatttened && npm run test:unflattened && npm run test:rest", "test:flatttened": "cp -f extensions/test-params-flatten-nested-true.local.env functions/.env && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithFlattening\" --testRegex=\"backfill.spec\"'", "test:unflattened": "cp -f extensions/test-params-flatten-nested-false.local.env functions/.env && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'", - "test:subcollection": "jest --testRegex=\"writeLogging\" --testRegex=\"Subcollection\" --detectOpenHandles", + "test:rest": "jest --testRegex=\"WithBuffer\" --testRegex=\"processBuffer.spec\" --testRegex=\"writeLogging\" --testRegex=\"Subcollection\" --detectOpenHandles", "typesenseServer": "docker compose up", "lint:fix": "eslint . --fix", "lint": "eslint .", @@ -23,15 +23,15 @@ "dotenv": "^16.4.7", "eslint": "^8.52.0", "eslint-config-google": "^0.14.0", + "eslint-config-prettier": "^10.0.1", "firebase-admin": "^13.0.2", - "firebase-functions": "^6.2.0", + "firebase-functions": "^6.3.2", "firebase-functions-test": "^3.4.0", - "eslint-config-prettier": "^10.0.1", "jest": "^29.7.0", "jest-dev-server": "^10.1.0", "jest-junit": "^16.0.0", "prettier": "^3.4.2", - "typesense": "^1.8.2" + "typesense": "^2.1.0-3" }, "private": true } diff --git a/test/backfill.spec.js b/test/backfill.spec.js index 0dbe1ff..6c4c3b5 100644 --- a/test/backfill.spec.js +++ b/test/backfill.spec.js @@ -1,4 +1,5 @@ const firebase = require("firebase-admin"); + const config = require("../functions/src/config.js"); const typesense = require("../functions/src/createTypesenseClient.js")(); @@ -16,7 +17,7 @@ describe("backfill", () => { // Clear any previously created collections try { - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections(config.typesenseCollectionName).delete(); } catch (e) { console.info(`${config.typesenseCollectionName} collection not found, proceeding...`); } @@ -46,7 +47,7 @@ describe("backfill", () => { // The above will automatically add the document to Typesense, // so delete it so we can test backfill - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections(config.typesenseCollectionName).delete(); await typesense.collections().create({ name: config.typesenseCollectionName, fields: [{name: ".*", type: "auto"}], @@ -57,7 +58,7 @@ describe("backfill", () => { await new Promise((r) => setTimeout(r, 2000)); // Check that the data was backfilled - const typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + const typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); expect(typesenseDocs[0]).toStrictEqual({ @@ -82,7 +83,7 @@ describe("backfill", () => { // The above will automatically add the document to Typesense, // so delete it so we can test backfill - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections(config.typesenseCollectionName).delete(); await typesense.collections().create({ name: config.typesenseCollectionName, fields: [{name: ".*", type: "auto"}], @@ -99,7 +100,7 @@ describe("backfill", () => { await new Promise((r) => setTimeout(r, 2000)); // Check that the data was backfilled - const typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + const typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); expect(typesenseDocs[0]).toStrictEqual({ @@ -124,7 +125,7 @@ describe("backfill", () => { // The above will automatically add the document to Typesense, // so delete it so we can test backfill - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections(config.typesenseCollectionName).delete(); await typesense.collections().create({ name: config.typesenseCollectionName, fields: [{name: ".*", type: "auto"}], @@ -141,7 +142,7 @@ describe("backfill", () => { await new Promise((r) => setTimeout(r, 2000)); // Check that the data was not backfilled - const typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + const typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); expect(typesenseDocsStr).toEqual(""); }, ); diff --git a/test/backfillSubcollection.spec.js b/test/backfillSubcollection.spec.js index 5d7b9d1..eacc83b 100644 --- a/test/backfillSubcollection.spec.js +++ b/test/backfillSubcollection.spec.js @@ -74,7 +74,7 @@ describe("backfillSubcollection", () => { // The above will automatically add the document to Typesense, // so delete it so we can test backfill - await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).delete(); + await testEnvironment.typesense.collections(testEnvironment.config.typesenseCollectionName).delete(); await testEnvironment.typesense.collections().create({ name: testEnvironment.config.typesenseCollectionName, fields: [{name: ".*", type: "auto"}], @@ -85,7 +85,7 @@ describe("backfillSubcollection", () => { await new Promise((r) => setTimeout(r, 2000)); // Check that the data was backfilled - const typesenseDocsStr = await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).documents().export(); + const typesenseDocsStr = await testEnvironment.typesense.collections(testEnvironment.config.typesenseCollectionName).documents().export(); const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -125,7 +125,7 @@ describe("backfillSubcollection", () => { // The above will automatically add the document to Typesense, // so delete it so we can test backfill - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections(config.typesenseCollectionName).delete(); await typesense.collections().create({ name: config.typesenseCollectionName, fields: [{name: ".*", type: "auto"}], @@ -142,7 +142,7 @@ describe("backfillSubcollection", () => { await new Promise((r) => setTimeout(r, 2000)); // Check that the data was backfilled - const typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + const typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); console.log(typesenseDocs); expect(typesenseDocs.length).toBe(1); @@ -180,7 +180,7 @@ describe("backfillSubcollection", () => { // The above will automatically add the document to Typesense, // so delete it so we can test backfill - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections(config.typesenseCollectionName).delete(); await typesense.collections().create({ name: config.typesenseCollectionName, fields: [{name: ".*", type: "auto"}], @@ -197,7 +197,7 @@ describe("backfillSubcollection", () => { await new Promise((r) => setTimeout(r, 2000)); // Check that the data was not backfilled - const typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + const typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); expect(typesenseDocsStr).toEqual(""); // Check that the error was logged @@ -256,7 +256,7 @@ describe("backfillSubcollection", () => { // The above will automatically add the document to Typesense, // so delete it so we can test backfill - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections(config.typesenseCollectionName).delete(); await typesense.collections().create({ name: config.typesenseCollectionName, fields: [{name: ".*", type: "auto"}], @@ -273,7 +273,7 @@ describe("backfillSubcollection", () => { await new Promise((r) => setTimeout(r, 2000)); // Check that the data was backfilled - const typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + const typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); diff --git a/test/indexOnWriteSubcollection.spec.js b/test/indexOnWriteSubcollection.spec.js index 8fe8036..3da980d 100644 --- a/test/indexOnWriteSubcollection.spec.js +++ b/test/indexOnWriteSubcollection.spec.js @@ -78,7 +78,7 @@ describe("indexOnWriteSubcollection", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was indexed - let typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export({exclude_fields: ""}); + let typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export({exclude_fields: ""}); let typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -97,7 +97,7 @@ describe("indexOnWriteSubcollection", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was updated - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export({exclude_fields: ""}); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export({exclude_fields: ""}); typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -115,7 +115,7 @@ describe("indexOnWriteSubcollection", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the subcollection document was deleted - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export({exclude_fields: ""}); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export({exclude_fields: ""}); expect(typesenseDocsStr).toBe(""); }); diff --git a/test/indexOnWriteSubcollectionWithBuffer.spec.js b/test/indexOnWriteSubcollectionWithBuffer.spec.js new file mode 100644 index 0000000..2580211 --- /dev/null +++ b/test/indexOnWriteSubcollectionWithBuffer.spec.js @@ -0,0 +1,248 @@ +const {TestEnvironment} = require("./support/testEnvironment"); + +const TEST_FIRESTORE_PARENT_COLLECTION_PATH = "users"; +const TEST_FIRESTORE_CHILD_FIELD_NAME = "books"; + +const TEST_TYPESENSE_FIELDS = [ + {name: "author", type: "string"}, + {name: "title", type: "string"}, +]; + +describe("indexOnWriteSubcollectionWithBuffer", () => { + let testEnvironment; + + const parentCollectionPath = TEST_FIRESTORE_PARENT_COLLECTION_PATH; + const childFieldName = TEST_FIRESTORE_CHILD_FIELD_NAME; + let parentIdField = null; + + let config = null; + let firestore = null; + + beforeAll((done) => { + testEnvironment = new TestEnvironment({ + dotenvPath: "extensions/test-params-subcategory-buffer-enabled.local.env", + outputAllEmulatorLogs: true, + typesenseFields: TEST_TYPESENSE_FIELDS, + }); + testEnvironment.setupTestEnvironment((err) => { + const matches = testEnvironment.config.firestoreCollectionPath.match(/{([^}]+)}/g); + expect(matches).toBeDefined(); + expect(matches.length).toBe(1); + + parentIdField = matches[0].replace(/{|}/g, ""); + expect(parentIdField).toBeDefined(); + + config = testEnvironment.config; + firestore = testEnvironment.firestore; + done(); + }); + }); + + afterAll(async () => { + await testEnvironment.teardownTestEnvironment(); + }); + + beforeEach(async () => { + await firestore.recursiveDelete(firestore.collection(parentCollectionPath)); + await firestore.recursiveDelete(firestore.collection(config.typesenseBufferCollectionInFirestore)); + await testEnvironment.clearAllData(); + }); + + describe("Subcollection writes with buffering", () => { + it("adds subcollection document to buffer when created", async () => { + const parentDocData = { + name: "Parent User", + age: 30, + }; + const parentDocRef = await firestore.collection(parentCollectionPath).add(parentDocData); + + const subDocData = { + author: "Subcollection Author", + title: "Subcollection Title", + nested_field: { + field1: "value1", + field2: ["value2", "value3", "value4"], + }, + }; + const subDocRef = await parentDocRef.collection(childFieldName).add(subDocData); + + await new Promise((r) => setTimeout(r, 2500)); + + const bufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", subDocRef.id).where("type", "==", "upsert").get(); + + expect(bufferSnapshot.empty).toBe(false); + const bufferDoc = bufferSnapshot.docs[0].data(); + expect(bufferDoc.status).toBe("pending"); + expect(bufferDoc.document).toMatchObject(subDocData); + expect(bufferDoc.documentId).toBe(subDocRef.id); + + expect(bufferDoc.pathParams).toBeDefined(); + expect(bufferDoc.pathParams[parentIdField]).toBe(parentDocRef.id); + }); + + it("adds subcollection document to buffer when updated", async () => { + const parentDocRef = await firestore.collection(parentCollectionPath).add({ + name: "Parent User", + }); + + const originalSubDocData = { + author: "Original Author", + title: "Original Title", + }; + const subDocRef = await parentDocRef.collection(childFieldName).add(originalSubDocData); + + await new Promise((r) => setTimeout(r, 2500)); + + const updatedSubDocData = { + author: "Updated Author", + title: "Updated Title", + additionalField: "New field", + }; + await subDocRef.update(updatedSubDocData); + + await new Promise((r) => setTimeout(r, 2500)); + + const bufferSnapshot = await firestore + .collection(config.typesenseBufferCollectionInFirestore) + .where("documentId", "==", subDocRef.id) + .where("type", "==", "upsert") + .orderBy("timestamp", "desc") + .limit(1) + .get(); + + expect(bufferSnapshot.empty).toBe(false); + const bufferDoc = bufferSnapshot.docs[0].data(); + expect(bufferDoc.status).toBe("pending"); + expect(bufferDoc.document).toMatchObject(updatedSubDocData); + expect(bufferDoc.documentId).toBe(subDocRef.id); + + expect(bufferDoc.pathParams).toBeDefined(); + expect(bufferDoc.pathParams[parentIdField]).toBe(parentDocRef.id); + }); + + it("adds delete operation to buffer when subcollection document is deleted", async () => { + const parentDocRef = await firestore.collection(parentCollectionPath).add({ + name: "Parent User for Delete Test", + }); + + const subDocData = { + author: "Delete Subcollection Test", + title: "Will Be Deleted", + }; + const subDocRef = await parentDocRef.collection(childFieldName).add(subDocData); + + await new Promise((r) => setTimeout(r, 2500)); + + await subDocRef.delete(); + + await new Promise((r) => setTimeout(r, 2500)); + + const bufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", subDocRef.id).where("type", "==", "delete").get(); + + expect(bufferSnapshot.empty).toBe(false); + const bufferDoc = bufferSnapshot.docs[0].data(); + expect(bufferDoc.status).toBe("pending"); + expect(bufferDoc.documentId).toBe(subDocRef.id); + expect(bufferDoc.type).toBe("delete"); + + expect(bufferDoc.pathParams).toBeDefined(); + expect(bufferDoc.pathParams[parentIdField]).toBe(parentDocRef.id); + }); + + it("adds documents with nested fields correctly to buffer", async () => { + const parentDocRef = await firestore.collection(parentCollectionPath).add({ + name: "Parent with Nested Fields", + }); + + const complexSubDocData = { + author: "Complex Author", + title: "Complex Title", + nested_object: { + level1: { + level2: { + level3: "Deep nested value", + array: [1, 2, 3], + }, + sibling: "Sibling value", + }, + tags: ["fiction", "adventure", "bestseller"], + }, + publishing: { + date: new Date("2023-01-01"), + publisher: "Test Publisher", + }, + }; + + const subDocRef = await parentDocRef.collection(childFieldName).add(complexSubDocData); + + await new Promise((r) => setTimeout(r, 2500)); + + const bufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", subDocRef.id).where("type", "==", "upsert").get(); + + expect(bufferSnapshot.empty).toBe(false); + const bufferDoc = bufferSnapshot.docs[0].data(); + + expect(bufferDoc.status).toBe("pending"); + expect(bufferDoc.document).toBeDefined(); + expect(bufferDoc.document.nested_object).toBeDefined(); + expect(bufferDoc.document.nested_object.level1.level2.level3).toBe("Deep nested value"); + expect(bufferDoc.document.nested_object.level1.level2.array).toEqual([1, 2, 3]); + expect(bufferDoc.document.nested_object.tags).toEqual(["fiction", "adventure", "bestseller"]); + + expect(bufferDoc.pathParams).toBeDefined(); + expect(bufferDoc.pathParams[parentIdField]).toBe(parentDocRef.id); + }); + + it("handles subcollection access after parent deletion", async () => { + const parentDocRef = await firestore.collection(parentCollectionPath).add({ + name: "Parent to Delete", + }); + + const subDocRefs = []; + for (let i = 0; i < 3; i++) { + const subDocRef = await parentDocRef.collection(childFieldName).add({ + author: `Author ${i}`, + title: `Title ${i}`, + }); + subDocRefs.push(subDocRef); + } + + await new Promise((r) => setTimeout(r, 2500)); + + await parentDocRef.delete(); + + await new Promise((r) => setTimeout(r, 2500)); + + const parentDocSnapshot = await parentDocRef.get(); + expect(parentDocSnapshot.exists).toBe(false); + + // In Firestore, deleting a parent document does NOT automatically delete subcollection documents + // Subcollection documents can still be accessed via their full path + for (const subDocRef of subDocRefs) { + const subDocSnapshot = await subDocRef.get(); + expect(subDocSnapshot.exists).toBe(true); + } + + // NOTE: The indexOnWrite function is configured to listen only on a specific collection path + // It wouldn't be triggered for parent document deletions since they're on a different path + // So we don't expect any buffer entries for the parent document deletion + + // If we later delete a subcollection document directly, it should still add to buffer + if (subDocRefs.length > 0) { + const subDocRefToDelete = subDocRefs[0]; + await subDocRefToDelete.delete(); + + await new Promise((r) => setTimeout(r, 2500)); + + const subDocBufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", subDocRefToDelete.id).where("type", "==", "delete").get(); + + expect(subDocBufferSnapshot.empty).toBe(false); + const subDocBufferDoc = subDocBufferSnapshot.docs[0].data(); + expect(subDocBufferDoc.status).toBe("pending"); + expect(subDocBufferDoc.type).toBe("delete"); + expect(subDocBufferDoc.pathParams).toBeDefined(); + expect(subDocBufferDoc.pathParams[parentIdField]).toBe(parentDocRef.id); + } + }); + }); +}); diff --git a/test/indexOnWriteWithBuffer.spec.js b/test/indexOnWriteWithBuffer.spec.js new file mode 100644 index 0000000..63ec6e4 --- /dev/null +++ b/test/indexOnWriteWithBuffer.spec.js @@ -0,0 +1,132 @@ +const {TestEnvironment} = require("./support/testEnvironment"); + +const TEST_FIRESTORE_PARENT_COLLECTION_PATH = "users"; + +const TEST_TYPESENSE_FIELDS = [ + {name: "author", type: "string"}, + {name: "title", type: "string"}, +]; + +describe("indexOnWriteWithBuffer", () => { + let testEnvironment; + + const parentCollectionPath = TEST_FIRESTORE_PARENT_COLLECTION_PATH; + + let config = null; + let firestore = null; + + beforeAll((done) => { + testEnvironment = new TestEnvironment({ + dotenvPath: "extensions/test-params-buffer-enabled.local.env", + outputAllEmulatorLogs: true, + typesenseFields: TEST_TYPESENSE_FIELDS, + }); + testEnvironment.setupTestEnvironment((err) => { + config = testEnvironment.config; + firestore = testEnvironment.firestore; + + done(); + }); + }); + + afterAll(async () => { + await testEnvironment.teardownTestEnvironment(); + }); + + beforeEach(async () => { + await firestore.recursiveDelete(firestore.collection(parentCollectionPath)); + await firestore.recursiveDelete(firestore.collection(config.typesenseBufferCollectionInFirestore)); + await testEnvironment.clearAllData(); + }); + + describe("Regular collection writes with buffering", () => { + it("adds document to buffer on create and update", async () => { + const docData = { + author: "Author A", + title: "Title X", + }; + + const docRef = await firestore.collection(config.firestoreCollectionPath).add(docData); + + await new Promise((r) => setTimeout(r, 2500)); + + let bufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", docRef.id).where("type", "==", "upsert").get(); + + expect(bufferSnapshot.empty).toBe(false); + let bufferDoc = bufferSnapshot.docs[0].data(); + expect(bufferDoc.status).toBe("pending"); + expect(bufferDoc.document).toMatchObject(docData); + expect(bufferDoc.documentId).toBe(docRef.id); + + const updatedData = { + author: "Author A Updated", + title: "Title X Updated", + }; + await docRef.update(updatedData); + + await new Promise((r) => setTimeout(r, 2500)); + + bufferSnapshot = await firestore + .collection(config.typesenseBufferCollectionInFirestore) + .where("documentId", "==", docRef.id) + .where("type", "==", "upsert") + .orderBy("timestamp", "desc") + .limit(1) + .get(); + + expect(bufferSnapshot.empty).toBe(false); + bufferDoc = bufferSnapshot.docs[0].data(); + expect(bufferDoc.status).toBe("pending"); + expect(bufferDoc.document).toMatchObject(updatedData); + expect(bufferDoc.documentId).toBe(docRef.id); + }); + + it("adds delete operation to buffer when document is deleted", async () => { + const docData = { + author: "Delete Test Author", + title: "Delete Test Title", + }; + + const docRef = await firestore.collection(config.firestoreCollectionPath).add(docData); + + await new Promise((r) => setTimeout(r, 2500)); + + await docRef.delete(); + + await new Promise((r) => setTimeout(r, 2500)); + + const bufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", docRef.id).where("type", "==", "delete").get(); + + expect(bufferSnapshot.empty).toBe(false); + const bufferDoc = bufferSnapshot.docs[0].data(); + expect(bufferDoc.status).toBe("pending"); + expect(bufferDoc.documentId).toBe(docRef.id); + expect(bufferDoc.type).toBe("delete"); + }); + }); + + describe("Buffer format validation", () => { + it("includes correct timestamp and retry information in buffer documents", async () => { + const docData = { + author: "Time Test Author", + title: "Time Test Title", + }; + + const docRef = await firestore.collection(config.firestoreCollectionPath).add(docData); + + await new Promise((r) => setTimeout(r, 2500)); + + const bufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", docRef.id).get(); + + expect(bufferSnapshot.empty).toBe(false); + const bufferDoc = bufferSnapshot.docs[0].data(); + + expect(bufferDoc.documentId).toBe(docRef.id); + expect(bufferDoc.document).toMatchObject(docData); + expect(bufferDoc.type).toBe("upsert"); + expect(bufferDoc.status).toBe("pending"); + expect(bufferDoc.timestamp).toBeDefined(); + expect(bufferDoc.retries).toBe(0); + }); + }); +}); diff --git a/test/indexOnWriteWithFlattening.spec.js b/test/indexOnWriteWithFlattening.spec.js index 35d6705..cbe8876 100644 --- a/test/indexOnWriteWithFlattening.spec.js +++ b/test/indexOnWriteWithFlattening.spec.js @@ -16,7 +16,7 @@ describe("indexOnWrite", () => { // delete the Typesense collection try { - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections(config.typesenseCollectionName).delete(); } catch (e) { console.info(`${config.typesenseCollectionName} collection not found, proceeding...`); } @@ -43,7 +43,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was indexed - let typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + let typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); let typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -58,7 +58,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was updated - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -71,7 +71,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was deleted - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); expect(typesenseDocsStr).toBe(""); }); @@ -86,7 +86,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was indexed - let typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + let typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); let typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -101,7 +101,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was updated - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -114,7 +114,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was deleted - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); expect(typesenseDocsStr).toBe(""); }); @@ -129,7 +129,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was indexed - let typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + let typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); let typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -144,7 +144,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was updated - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -157,7 +157,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was deleted - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); expect(typesenseDocsStr).toBe(""); }); @@ -182,7 +182,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was indexed - let typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + let typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); let typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -203,7 +203,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was updated - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -222,7 +222,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was deleted - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); expect(typesenseDocsStr).toBe(""); }); @@ -238,7 +238,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was indexed - let typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + let typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); let typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -253,7 +253,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was updated - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -266,7 +266,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was deleted - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); expect(typesenseDocsStr).toBe(""); }); @@ -281,7 +281,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was indexed - let typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + let typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -294,7 +294,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was deleted - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); expect(typesenseDocsStr).toBe(""); }); @@ -309,7 +309,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was indexed - let typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + let typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); let typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -327,7 +327,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was updated - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -343,7 +343,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was deleted - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); expect(typesenseDocsStr).toBe(""); }); @@ -358,7 +358,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was indexed - let typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + let typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); let typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -376,7 +376,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was updated - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -392,7 +392,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was deleted - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); expect(typesenseDocsStr).toBe(""); }); @@ -407,7 +407,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was indexed - let typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + let typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); let typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -422,7 +422,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was updated - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -435,7 +435,7 @@ describe("indexOnWrite", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was deleted - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); expect(typesenseDocsStr).toBe(""); }); diff --git a/test/indexOnWriteWithoutFlattening.spec.js b/test/indexOnWriteWithoutFlattening.spec.js index 1e9209e..5e9a865 100644 --- a/test/indexOnWriteWithoutFlattening.spec.js +++ b/test/indexOnWriteWithoutFlattening.spec.js @@ -16,7 +16,7 @@ describe("indexOnWriteWithoutFlattening", () => { // delete the Typesense collection try { - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections(config.typesenseCollectionName).delete(); } catch (e) { console.info(`${config.typesenseCollectionName} collection not found, proceeding...`); } @@ -56,7 +56,7 @@ describe("indexOnWriteWithoutFlattening", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was indexed - let typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + let typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); let typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -74,7 +74,7 @@ describe("indexOnWriteWithoutFlattening", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was updated - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export({exclude_fields: ""}); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export({exclude_fields: ""}); typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); @@ -90,7 +90,7 @@ describe("indexOnWriteWithoutFlattening", () => { await new Promise((r) => setTimeout(r, 2500)); // check that the document was deleted - typesenseDocsStr = await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).documents().export(); + typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); expect(typesenseDocsStr).toBe(""); }); diff --git a/test/processBuffer.spec.js b/test/processBuffer.spec.js new file mode 100644 index 0000000..c7c19fd --- /dev/null +++ b/test/processBuffer.spec.js @@ -0,0 +1,207 @@ +const {TestEnvironment} = require("./support/testEnvironment"); + +const TEST_FIRESTORE_PARENT_COLLECTION_PATH = "users"; + +const TEST_TYPESENSE_FIELDS = [ + {name: "author", type: "string"}, + {name: "title", type: "string"}, +]; + +describe("processBuffer", () => { + let testEnvironment; + + const parentCollectionPath = TEST_FIRESTORE_PARENT_COLLECTION_PATH; + + let config = null; + let firestore = null; + let typesense = null; + + beforeAll((done) => { + testEnvironment = new TestEnvironment({ + dotenvPath: "extensions/test-params-flatten-nested-false.local.env", + outputAllEmulatorLogs: true, + typesenseFields: TEST_TYPESENSE_FIELDS, + }); + testEnvironment.setupTestEnvironment((err) => { + config = testEnvironment.config; + firestore = testEnvironment.firestore; + typesense = testEnvironment.typesense; + done(); + }); + }); + + afterAll(async () => { + await testEnvironment.teardownTestEnvironment(); + }); + + beforeEach(async () => { + await firestore.recursiveDelete(firestore.collection(parentCollectionPath)); + + await firestore.recursiveDelete(firestore.collection(config.typesenseBufferCollectionInFirestore)); + + await testEnvironment.clearAllData(); + }); + + describe("Processing buffered operations", () => { + it("processes pending upsert operations from the buffer collection", async () => { + const documentId = "test-doc-1"; + const bookData = { + author: "Author A", + title: "Title X", + }; + + await firestore.collection(config.typesenseBufferCollectionInFirestore).add({ + documentId: documentId, + document: bookData, + type: "upsert", + status: "pending", + timestamp: Date.now(), + retries: 0, + }); + + const {processTypesenseBuffer} = require("../functions/src/processBuffer"); + await processTypesenseBuffer(); + + await new Promise((r) => setTimeout(r, 2500)); + + const bufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", documentId).get(); + + expect(bufferSnapshot.empty).toBe(false); + expect(bufferSnapshot.docs[0].data().status).toBe("completed"); + + const typesenseDocsStr = await typesense.collections(config.typesenseCollectionName).documents().export(); + const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); + + expect(typesenseDocs.length).toBe(1); + expect(typesenseDocs[0].id).toBe(documentId); + expect(typesenseDocs[0].author).toBe(bookData.author); + expect(typesenseDocs[0].title).toBe(bookData.title); + }); + + it("processes pending delete operations from the buffer collection", async () => { + const documentId = "test-doc-to-delete"; + const documentData = { + id: documentId, + author: "Author B", + title: "Title Y", + }; + + await typesense.collections(config.typesenseCollectionName).documents().create(documentData); + + const typesenseDocsStrOriginal = await typesense.collections(config.typesenseCollectionName).documents().export(); + const typesenseDocs = typesenseDocsStrOriginal.split("\n").map((s) => JSON.parse(s)); + expect(typesenseDocs.length).toBe(1); + + await firestore.collection(config.typesenseBufferCollectionInFirestore).add({ + documentId: documentId, + type: "delete", + status: "pending", + timestamp: Date.now(), + retries: 0, + }); + + const {processTypesenseBuffer} = require("../functions/src/processBuffer"); + await processTypesenseBuffer(); + + await new Promise((r) => setTimeout(r, 2500)); + + const bufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", documentId).get(); + + expect(bufferSnapshot.empty).toBe(false); + expect(bufferSnapshot.docs[0].data().status).toBe("completed"); + + const typesenseDocsStrAfterDelete = await typesense.collections(config.typesenseCollectionName).documents().export(); + expect(typesenseDocsStrAfterDelete).toBe(""); // Empty means no documents + }); + + it("handles retry logic for failed deletions", async () => { + const missingDocId = "malformed-doc"; + + const documentData = { + id: "test-doc-to-delete", + author: "Author B", + title: "Title Y", + }; + + await typesense.collections(config.typesenseCollectionName).documents().create(documentData); + await firestore.collection(config.typesenseBufferCollectionInFirestore).add({ + documentId: missingDocId, + type: "delete", + status: "pending", + timestamp: Date.now(), + retries: 0, + }); + await firestore.collection(config.typesenseBufferCollectionInFirestore).add({ + documentId: documentData.id, + document: documentData, + type: "delete", + status: "pending", + timestamp: Date.now(), + retries: 0, + }); + + const {processTypesenseBuffer} = require("../functions/src/processBuffer"); + for (let i = 0; i < config.typesenseBufferMaxRetries; i++) { + await processTypesenseBuffer(); + + const bufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", missingDocId).get(); + + expect(bufferSnapshot.empty).toBe(false); + const docData = bufferSnapshot.docs[0].data(); + + console.dir(docData, {depth: null}); + expect(docData.status).toBe("retrying"); + expect(docData.retries).toBe(i + 1); + } + + await processTypesenseBuffer(); + await new Promise((r) => setTimeout(r, 2500)); + const bufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", missingDocId).get(); + + expect(bufferSnapshot.empty).toBe(false); + const docData = bufferSnapshot.docs[0].data(); + + expect(docData.status).toBe("failed"); + expect(docData.retries).toBe(config.typesenseBufferMaxRetries); + expect(docData.lastError).toBeDefined(); + }); + + it("handles retry logic for failed upserts", async () => { + const documentId = "malformed-doc"; + + await firestore.collection(config.typesenseBufferCollectionInFirestore).add({ + documentId: documentId, + // Intentionally missing the document field for upsert to cause failure + type: "upsert", + status: "pending", + timestamp: Date.now(), + retries: 0, + }); + + const {processTypesenseBuffer} = require("../functions/src/processBuffer"); + for (let i = 0; i < config.typesenseBufferMaxRetries; i++) { + await processTypesenseBuffer(); + + const bufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", documentId).get(); + + expect(bufferSnapshot.empty).toBe(false); + const docData = bufferSnapshot.docs[0].data(); + + console.dir(docData, {depth: null}); + expect(docData.status).toBe("retrying"); + expect(docData.retries).toBe(i + 1); + } + + await processTypesenseBuffer(); + await new Promise((r) => setTimeout(r, 2500)); + const bufferSnapshot = await firestore.collection(config.typesenseBufferCollectionInFirestore).where("documentId", "==", documentId).get(); + + expect(bufferSnapshot.empty).toBe(false); + const docData = bufferSnapshot.docs[0].data(); + + expect(docData.status).toBe("failed"); + expect(docData.retries).toBe(config.typesenseBufferMaxRetries); + expect(docData.lastError).toBeDefined(); + }); + }); +}); diff --git a/test/support/testEnvironment.js b/test/support/testEnvironment.js index 70278f5..966a038 100644 --- a/test/support/testEnvironment.js +++ b/test/support/testEnvironment.js @@ -47,6 +47,7 @@ class TestEnvironment { shouldOutputAllEmulatorLogs = false; dotenvPath = null; dotenvConfig = null; + typesenseFields = null; // Emulator vars emulator = null; @@ -65,10 +66,11 @@ class TestEnvironment { * @param {string} config.dotenvConfig - path to the env file to use for the firebase emulator and test * @param {boolean} config.debugLog - whether to log all emulator logs to console */ - constructor({dotenvPath, dotenvConfig, outputAllEmulatorLogs = false} = {}) { + constructor({dotenvPath, dotenvConfig, outputAllEmulatorLogs = false, typesenseFields = null} = {}) { this.dotenvPath = dotenvPath; this.dotenvConfig = dotenvConfig; this.shouldOutputAllEmulatorLogs = outputAllEmulatorLogs; + this.typesenseFields = typesenseFields; if (dotenvPath && dotenvConfig) { throw new Error("Provide either 'dotenvPath' or 'dotenvConfig', not both."); @@ -213,13 +215,16 @@ class TestEnvironment { } try { - await this.typesense.collections(encodeURIComponent(this.config.typesenseCollectionName)).delete(); + await this.typesense.collections(this.config.typesenseCollectionName).delete(); } catch (e) { directConsole.info(`${this.config.typesenseCollectionName} collection not found, proceeding...`); } + + const fields = this.typesenseFields || [{name: ".*", type: "auto"}]; + await this.typesense.collections().create({ name: this.config.typesenseCollectionName, - fields: [{name: ".*", type: "auto"}], + fields, enable_nested_fields: true, }); } diff --git a/test/writeLogging.spec.js b/test/writeLogging.spec.js index daefc11..92d54d5 100644 --- a/test/writeLogging.spec.js +++ b/test/writeLogging.spec.js @@ -50,7 +50,7 @@ describe("indexOnWriteLogging - when shouldLogTypesenseInserts is false", () => // The above will automatically add the document to Typesense, // so delete it so we can test backfill - await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).delete(); + await testEnvironment.typesense.collections(testEnvironment.config.typesenseCollectionName).delete(); await testEnvironment.typesense.collections().create({ name: testEnvironment.config.typesenseCollectionName, fields: [{name: ".*", type: "auto"}], @@ -61,7 +61,7 @@ describe("indexOnWriteLogging - when shouldLogTypesenseInserts is false", () => await new Promise((r) => setTimeout(r, 2000)); // Check that the data was backfilled - const typesenseDocsStr = await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).documents().export(); + const typesenseDocsStr = await testEnvironment.typesense.collections(testEnvironment.config.typesenseCollectionName).documents().export(); const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); expect(typesenseDocs[0]).toStrictEqual({ @@ -136,7 +136,7 @@ TYPESENSE_API_KEY=xyz // The above will automatically add the document to Typesense, // so delete it so we can test backfill - await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).delete(); + await testEnvironment.typesense.collections(testEnvironment.config.typesenseCollectionName).delete(); await testEnvironment.typesense.collections().create({ name: testEnvironment.config.typesenseCollectionName, fields: [{name: ".*", type: "auto"}], @@ -147,7 +147,7 @@ TYPESENSE_API_KEY=xyz await new Promise((r) => setTimeout(r, 2000)); // Check that the data was backfilled - const typesenseDocsStr = await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).documents().export(); + const typesenseDocsStr = await testEnvironment.typesense.collections(testEnvironment.config.typesenseCollectionName).documents().export(); const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); expect(typesenseDocs.length).toBe(1); const expectedResult = {