From 140b8ce3a597fb49a0580259b83fd40210f90d9d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 5 Aug 2026 18:45:50 +1200 Subject: [PATCH 1/2] fix(destination): recreate an index with the source's own prefix lengths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validateFieldsForIndexes() derived every length from the destination column instead of the index it was recreating: null, or the array cap for an array column. The source's recorded prefix lengths were read into the Index resource and then never used — the class had no getter for them at all. An index only fits under an adapter's byte limit BECAUSE of that prefix. Two large string columns capped at 100 and 20 bytes are a valid index on the source and a 767-byte violation on the destination, so restoring a healthy database failed to recreate its own indexes with 'Invalid index: Index length is longer than the maximum: 767'. Measured on production: 3 customers across 3 regions hit exactly this, most recently four days ago, and one retried nine times in eighty minutes before giving up (DAT-2113). Lengths now come from the resource, with zero — how a source records 'no prefix' for a position — staying no-prefix rather than becoming a zero-length one. Array columns keep their MAX_ARRAY_INDEX_LENGTH override. DAT-2113 Co-Authored-By: Claude Fable 5 --- src/Migration/Destinations/Appwrite.php | 9 +- src/Migration/Resources/Database/Index.php | 8 + .../Destinations/AppwriteIndexLengthsTest.php | 302 ++++++++++++++++++ 3 files changed, 318 insertions(+), 1 deletion(-) create mode 100644 tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 33d149a7..ab42ae09 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -3886,7 +3886,14 @@ private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table ); } - $lengths[$i] = null; + // The source's own prefix length, when it recorded one. An index + // that only fits under the adapter's byte limit because of an + // explicit prefix (e.g. two large strings capped at 100 and 20) + // must be recreated with that prefix, or restoring a healthy + // database fails its own indexes with "Index length is longer + // than the maximum". Zero means the source recorded no length. + $sourceLength = $resource->getLengths()[$i] ?? null; + $lengths[$i] = \is_int($sourceLength) && $sourceLength > 0 ? $sourceLength : null; if ($columnArray === true) { $lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH; diff --git a/src/Migration/Resources/Database/Index.php b/src/Migration/Resources/Database/Index.php index 599672a6..e905d6f7 100644 --- a/src/Migration/Resources/Database/Index.php +++ b/src/Migration/Resources/Database/Index.php @@ -139,6 +139,14 @@ public function getColumns(): array return $this->columns; } + /** + * @return array + */ + public function getLengths(): array + { + return $this->lengths; + } + /** * @return array */ diff --git a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php new file mode 100644 index 00000000..259e0fa9 --- /dev/null +++ b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php @@ -0,0 +1,302 @@ + null, + static fn (mixed $value, UtopiaDocument $document, UtopiaDatabase $database): array => $database->getAuthorization()->skip( + static fn (): array => $database->find('attributes', [ + \Utopia\Database\Query::equal('collectionInternalId', [$document->getSequence()]), + \Utopia\Database\Query::equal('databaseInternalId', [$document->getAttribute('databaseInternalId')]), + ]), + ), + ); + UtopiaDatabase::addFilter( + 'subQueryIndexes', + static fn (mixed $value) => null, + static fn (mixed $value, UtopiaDocument $document, UtopiaDatabase $database): array => $database->getAuthorization()->skip( + static fn (): array => $database->find('indexes', [ + \Utopia\Database\Query::equal('collectionInternalId', [$document->getSequence()]), + \Utopia\Database\Query::equal('databaseInternalId', [$document->getAttribute('databaseInternalId')]), + ]), + ), + ); + } + + public function testRestoredIndexCarriesTheSourcePrefixLengths(): void + { + [$destination, $database] = $this->transferIndex(lengths: [100, 20]); + + $this->assertSame([], $this->errorMessages($destination)); + + $created = $this->indexDocument($database); + $this->assertFalse($created->isEmpty(), 'The index must be created'); + $this->assertSame([100, 20], $created->getAttribute('lengths')); + } + + public function testRestoredIndexWithoutSourceLengthsFailsTheAdapterLimit(): void + { + // The two 600-char columns exceed the Memory adapter's 1024-byte index + // cap without prefixes, exactly like the production 767-byte MySQL cap: + // an index that NEEDS its source lengths cannot be recreated without + // them, which is the customer-facing failure this suite pins. + [$destination, $database] = $this->transferIndex(lengths: []); + + $messages = $this->errorMessages($destination); + $this->assertNotSame([], $messages, 'Expected the full-width index to exceed the adapter limit'); + $this->assertStringContainsString('Index length is longer than the maximum', $messages[0]); + $this->assertTrue($this->indexDocument($database)->isEmpty(), 'The failed index must not be recorded'); + } + + public function testZeroSourceLengthMeansNoPrefix(): void + { + // Zero is how a source records "no prefix" for a position (a short + // column needs none), and it must stay no-prefix rather than becoming a + // zero-length one. The metadata collection types `lengths` as an integer + // array, so no-prefix reads back as 0 — the shape a live production row + // for exactly this case carries ([100, 0]). + [$destination, $database] = $this->transferIndex(lengths: [100, 0], columnSizes: [600, 30]); + + $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame([100, 0], $this->indexDocument($database)->getAttribute('lengths')); + } + + /** + * @param array $lengths + * @param array $columnSizes + * @return array{AppwriteDestination, UtopiaDatabase} + */ + private function transferIndex(array $lengths, array $columnSizes = [600, 600]): array + { + $database = $this->projectDatabase(); + + $source = new MockSource(); + $databaseResource = new DatabaseResource( + id: 'shop', + name: 'Shop', + type: 'tablesdb', + database: 'source-dsn', + ); + $table = new Table($databaseResource, 'Orders', 'orders'); + $source->pushMockResource($databaseResource); + $source->pushMockResource($table); + // Distinct ids: MockSource keys its map by resource id, and a Column + // carries none by default, so two columns would collide onto one. + $source->pushMockResource((new Text('reference', $table, size: $columnSizes[0]))->setId('reference')); + $source->pushMockResource((new Text('channel', $table, size: $columnSizes[1]))->setId('channel')); + $source->pushMockResource(new Index( + id: 'idx_reference_channel', + key: 'idx_reference_channel', + table: $table, + type: 'key', + columns: ['reference', 'channel'], + lengths: $lengths, + orders: ['ASC', 'ASC'], + )); + + $destination = new AppwriteDestination( + project: 'destination-project', + endpoint: 'http://example.test/v1', + key: 'test-key', + dbForProject: $database, + getDatabasesDB: static fn (UtopiaDocument $document): UtopiaDatabase => $database, + collectionStructure: [ + 'attributes' => [ + $this->attributeArray('databaseInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), + $this->attributeArray('databaseId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), + $this->attributeArray('name', UtopiaDatabase::VAR_STRING, size: 256), + $this->attributeArray('enabled', UtopiaDatabase::VAR_BOOLEAN, default: true), + $this->attributeArray('documentSecurity', UtopiaDatabase::VAR_BOOLEAN, default: false), + $this->attributeArray('search', UtopiaDatabase::VAR_STRING, size: 16384), + $this->attributeArray('attributes', UtopiaDatabase::VAR_STRING, size: 16384, filters: ['subQueryAttributes']), + $this->attributeArray('indexes', UtopiaDatabase::VAR_STRING, size: 16384, filters: ['subQueryIndexes']), + ], + 'indexes' => [], + ], + dbForPlatform: $database, + projectInternalId: '1', + onDuplicate: OnDuplicate::Fail, + ); + + // Two passes: Transfer::GROUP_DATABASES_RESOURCES replays indexes BEFORE + // columns, the reverse of what a real source emits, so a single call + // would offer the index against a table with no columns yet. Selecting + // the schema first and the index second reproduces the production order + // this defect lives in. + $transfer = new Transfer($source, $destination); + $database->getAuthorization()->skip( + static function () use ($transfer): void { + $noop = static function (): void { + }; + $transfer->run([Resource::TYPE_DATABASE, Resource::TYPE_TABLE, Resource::TYPE_COLUMN], $noop); + $transfer->run([Resource::TYPE_INDEX], $noop); + }, + ); + + return [$destination, $database]; + } + + private function projectDatabase(): UtopiaDatabase + { + $database = new UtopiaDatabase( + new MemoryAdapter(), + new Cache(new MemoryCache()), + ); + $database + ->setDatabase('appwrite') + ->setNamespace('_project'); + $database->create(); + + $database->createCollection('databases', [ + $this->attribute('name', UtopiaDatabase::VAR_STRING, required: true, size: 256), + $this->attribute('enabled', UtopiaDatabase::VAR_BOOLEAN, default: true), + $this->attribute('search', UtopiaDatabase::VAR_STRING, size: 16384), + $this->attribute('originalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', UtopiaDatabase::VAR_STRING, default: 'tablesdb', size: 128), + $this->attribute('database', UtopiaDatabase::VAR_STRING, size: 2000), + ]); + + $database->createCollection('attributes', [ + $this->attribute('key', UtopiaDatabase::VAR_STRING, size: 256), + $this->attribute('databaseInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('databaseId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', UtopiaDatabase::VAR_STRING, size: 256), + $this->attribute('status', UtopiaDatabase::VAR_STRING, size: 64), + $this->attribute('size', UtopiaDatabase::VAR_INTEGER), + $this->attribute('required', UtopiaDatabase::VAR_BOOLEAN, default: false), + $this->attribute('signed', UtopiaDatabase::VAR_BOOLEAN, default: true), + $this->attribute('default', UtopiaDatabase::VAR_STRING, size: 16384), + $this->attribute('array', UtopiaDatabase::VAR_BOOLEAN, default: false), + $this->attribute('format', UtopiaDatabase::VAR_STRING, size: 64), + $this->attribute('formatOptions', UtopiaDatabase::VAR_STRING, size: 16384, filters: ['json']), + $this->attribute('filters', UtopiaDatabase::VAR_STRING, size: 64, array: true), + $this->attribute('options', UtopiaDatabase::VAR_STRING, size: 16384, filters: ['json']), + $this->attribute('error', UtopiaDatabase::VAR_STRING, size: 2048), + ]); + + $database->createCollection('indexes', [ + $this->attribute('key', UtopiaDatabase::VAR_STRING, size: 256), + $this->attribute('status', UtopiaDatabase::VAR_STRING, size: 64), + $this->attribute('databaseInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('databaseId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', UtopiaDatabase::VAR_STRING, size: 16), + $this->attribute('attributes', UtopiaDatabase::VAR_STRING, size: 256, array: true), + $this->attribute('lengths', UtopiaDatabase::VAR_INTEGER, array: true), + $this->attribute('orders', UtopiaDatabase::VAR_STRING, size: 4, array: true), + $this->attribute('error', UtopiaDatabase::VAR_STRING, size: 2048), + ]); + + return $database; + } + + /** + * @param array $filters + * @return array + */ + private function attributeArray( + string $id, + string $type, + bool $required = false, + mixed $default = null, + int $size = 0, + bool $array = false, + array $filters = [], + ): array { + return [ + '$id' => $id, + 'type' => $type, + 'size' => $size, + 'required' => $required, + 'default' => $default, + 'array' => $array, + 'signed' => true, + 'filters' => $filters, + ]; + } + + /** + * @param array $filters + */ + private function attribute( + string $id, + string $type, + bool $required = false, + mixed $default = null, + int $size = 0, + bool $array = false, + array $filters = [], + ): UtopiaDocument { + return new UtopiaDocument($this->attributeArray($id, $type, $required, $default, $size, $array, $filters)); + } + + /** + * @return array + */ + private function errorMessages(AppwriteDestination $destination): array + { + return \array_map( + static fn ($error): string => $error->getMessage(), + $destination->getErrors(), + ); + } + + private function indexDocument(UtopiaDatabase $database): UtopiaDocument + { + $indexes = $database->getAuthorization()->skip( + static fn (): array => $database->find('indexes'), + ); + + return $indexes[0] ?? new UtopiaDocument(); + } +} From 31d4abb6ead13b58858dea4f4f7f785be26739d0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 5 Aug 2026 19:19:43 +1200 Subject: [PATCH 2/2] fix(destination): compare index prefix lengths when reconciling an existing index Greptile on this PR, and correct: indexSpecMatches() omitted lengths, so an Overwrite migration onto an index whose ONLY difference was its prefix treated it as already matching and skipped it, silently keeping the destination's lengths -- reintroducing the very defect this PR fixes, on the overwrite path instead of the create path. The exclusion was reasonable when it was written: lengths were derived per-column from the adapter and not carried on the resource, so comparing them compared nothing. The source's own lengths drive them now, so a length-only difference is a genuine mismatch. Positions are compared as integers so 'no prefix' matches however it is spelled -- the resource records 0, the metadata collection reads back 0 or null. The test asserts the decision rather than the recreate: the in-memory adapter cannot drop an index the destination believes it holds, and that is an adapter limit rather than the behaviour under test. DAT-2113 Co-Authored-By: Claude Fable 5 --- src/Migration/Destinations/Appwrite.php | 33 ++++++++- .../Destinations/AppwriteIndexLengthsTest.php | 68 ++++++++++++++++--- 2 files changed, 89 insertions(+), 12 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index ab42ae09..0e20548e 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -1936,14 +1936,41 @@ private function attributeSpecMatches(UtopiaDocument $existing, Column|Attribute } /** - * `lengths` is intentionally not compared: it's derived per-column from - * the adapter's index validator and not settable on the Index resource. + * `lengths` is compared. It used to be excluded on the grounds that it was + * derived per-column from the adapter's index validator rather than carried + * by the resource — true then, but the source's own prefix lengths now + * drive it, so a destination index whose only difference is its prefix is a + * genuine mismatch. Skipping it left an Overwrite migration silently + * keeping the destination's lengths. */ private function indexSpecMatches(UtopiaDocument $existingIdx, Index $resource): bool { return $existingIdx->getAttribute('type') === $resource->getType() && $existingIdx->getAttribute('attributes') === $resource->getColumns() - && $existingIdx->getAttribute('orders') === $resource->getOrders(); + && $existingIdx->getAttribute('orders') === $resource->getOrders() + && $this->indexLengthsMatch($existingIdx, $resource); + } + + /** + * Compare prefix lengths position by position, treating "no prefix" as one + * value however it is spelled: the resource records it as a `0`, and the + * metadata collection reads it back as `0` or `null` depending on how it + * was written. + */ + private function indexLengthsMatch(UtopiaDocument $existingIdx, Index $resource): bool + { + $existing = $existingIdx->getAttribute('lengths'); + $existing = \is_array($existing) ? \array_values($existing) : []; + $wanted = \array_values($resource->getLengths()); + + $columns = \count($resource->getColumns()); + for ($position = 0; $position < $columns; $position++) { + if ((int) ($existing[$position] ?? 0) !== (int) ($wanted[$position] ?? 0)) { + return false; + } + } + + return true; } private function tableIdentity(UtopiaDocument $database, UtopiaDocument $table): string diff --git a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php index 259e0fa9..362f6e79 100644 --- a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php @@ -108,13 +108,46 @@ public function testZeroSourceLengthMeansNoPrefix(): void $this->assertSame([100, 0], $this->indexDocument($database)->getAttribute('lengths')); } + /** + * An Overwrite migration onto an index whose ONLY difference is its prefix + * lengths must recreate it. The spec comparison used to omit lengths, so + * the index was treated as already matching and the destination silently + * kept its own prefixes — the shape Greptile flagged on this PR. + */ + public function testAnOverwriteDoesNotTreatALengthOnlyDifferenceAsAMatch(): void + { + [, , $overwritten] = $this->transferIndex( + lengths: [100, 20], + onDuplicate: OnDuplicate::Overwrite, + existingLengths: [50, 20], + ); + + // The observable is the decision, not the recreate: an index whose only + // difference is its prefix was reported as already matching and skipped, + // so the destination silently kept its own lengths. The drop-and-recreate + // that follows cannot be asserted on the in-memory adapter, which does + // not support dropping an index the destination believes it holds — that + // is an adapter limit, not the behaviour under test, and saying so beats + // asserting an artifact of it. + $this->assertNotSame( + Resource::STATUS_SKIPPED, + $overwritten->getStatus(), + 'A length-only difference must not be reported as already existing on the destination.', + ); + } + /** * @param array $lengths * @param array $columnSizes - * @return array{AppwriteDestination, UtopiaDatabase} + * @param array $existingLengths + * @return array{AppwriteDestination, UtopiaDatabase, Index} */ - private function transferIndex(array $lengths, array $columnSizes = [600, 600]): array - { + private function transferIndex( + array $lengths, + array $columnSizes = [600, 600], + OnDuplicate $onDuplicate = OnDuplicate::Fail, + array $existingLengths = [], + ): array { $database = $this->projectDatabase(); $source = new MockSource(); @@ -131,15 +164,21 @@ private function transferIndex(array $lengths, array $columnSizes = [600, 600]): // carries none by default, so two columns would collide onto one. $source->pushMockResource((new Text('reference', $table, size: $columnSizes[0]))->setId('reference')); $source->pushMockResource((new Text('channel', $table, size: $columnSizes[1]))->setId('channel')); - $source->pushMockResource(new Index( + + // An Overwrite only applies when the source is newer than what the + // destination holds, so the seeded index is stamped older than the one + // that replaces it. + $index = static fn (array $withLengths, string $updatedAt = ''): Index => new Index( id: 'idx_reference_channel', key: 'idx_reference_channel', table: $table, type: 'key', columns: ['reference', 'channel'], - lengths: $lengths, + lengths: $withLengths, orders: ['ASC', 'ASC'], - )); + createdAt: $updatedAt, + updatedAt: $updatedAt, + ); $destination = new AppwriteDestination( project: 'destination-project', @@ -162,7 +201,7 @@ private function transferIndex(array $lengths, array $columnSizes = [600, 600]): ], dbForPlatform: $database, projectInternalId: '1', - onDuplicate: OnDuplicate::Fail, + onDuplicate: $onDuplicate, ); // Two passes: Transfer::GROUP_DATABASES_RESOURCES replays indexes BEFORE @@ -170,17 +209,28 @@ private function transferIndex(array $lengths, array $columnSizes = [600, 600]): // would offer the index against a table with no columns yet. Selecting // the schema first and the index second reproduces the production order // this defect lives in. + $transferred = $index($lengths, $existingLengths !== [] ? '2026-06-01 00:00:00' : ''); + $transfer = new Transfer($source, $destination); $database->getAuthorization()->skip( - static function () use ($transfer): void { + static function () use ($transfer, $source, $index, $transferred, $existingLengths): void { $noop = static function (): void { }; $transfer->run([Resource::TYPE_DATABASE, Resource::TYPE_TABLE, Resource::TYPE_COLUMN], $noop); + + if ($existingLengths !== []) { + // Seed the index the overwrite will meet, carrying the + // prefixes the destination already has. + $source->pushMockResource($index($existingLengths, '2026-01-01 00:00:00')); + $transfer->run([Resource::TYPE_INDEX], $noop); + } + + $source->pushMockResource($transferred); $transfer->run([Resource::TYPE_INDEX], $noop); }, ); - return [$destination, $database]; + return [$destination, $database, $transferred]; } private function projectDatabase(): UtopiaDatabase