From 890c9ee4a91efb2ae0987560385605ce14ba7377 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 5 Jan 2026 12:19:39 -0500 Subject: [PATCH 1/5] feat: publish and use nix-catalog --- .github/workflows/ami-release-nix.yml | 41 +++++++++++++++++++ .../pg_upgrade_scripts/initiate.sh | 39 +++++++++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 7748b98a5..507d0e03d 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -138,6 +138,47 @@ jobs: aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz + - name: Update nix store path catalog + run: | + VERSION="${{ steps.process_release_version.outputs.version }}" + GIT_SHA="${{ github.sha }}" + PG_VERSION="${{ matrix.postgres_version }}" + SYSTEM="aarch64-linux" + ATTR="psql_${PG_VERSION}/bin" + + # Get store path for this build + STORE_PATH=$(nix eval --raw ".#psql_${PG_VERSION}/bin.outPath") + + # Catalog keyed by git SHA (what the upgrade script has access to) + CATALOG_URL="https://supabase-public-artifacts-bucket.s3.amazonaws.com/nix-catalog/${GIT_SHA}.json" + CATALOG_S3="s3://supabase-public-artifacts-bucket/nix-catalog/${GIT_SHA}.json" + + # Download existing catalog or create new + if curl -sf "$CATALOG_URL" -o /tmp/catalog.json; then + echo "Updating existing catalog for ${GIT_SHA}" + else + echo '{}' > /tmp/catalog.json + echo "Creating new catalog for ${GIT_SHA}" + fi + + # Add metadata (version and git_sha) and entry for this system/attr + jq --arg ver "$VERSION" \ + --arg sha "$GIT_SHA" \ + --arg sys "$SYSTEM" \ + --arg attr "$ATTR" \ + --arg path "$STORE_PATH" \ + '.version = $ver | .git_sha = $sha | if .[$sys] == null then .[$sys] = {} else . end | .[$sys][$attr] = $path' \ + /tmp/catalog.json > /tmp/catalog-updated.json + + echo "Catalog entry: ${SYSTEM}.${ATTR} = ${STORE_PATH}" + cat /tmp/catalog-updated.json + + # Upload updated catalog + aws s3 cp /tmp/catalog-updated.json "$CATALOG_S3" \ + --content-type "application/json" + + echo "Catalog uploaded to ${CATALOG_S3}" + - name: Create release uses: softprops/action-gh-release@v2 with: diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 446cd2797..ac8798f53 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -301,11 +301,46 @@ EXTRA_NIX_CONF fi fi - echo "1.2. Installing flake revision: $NIX_FLAKE_VERSION" + echo "1.2. Fetching store path for flake revision: $NIX_FLAKE_VERSION" # shellcheck disable=SC1091 source /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh nix-collect-garbage -d > /tmp/pg_upgrade-nix-gc.log 2>&1 || true - PG_UPGRADE_BIN_DIR=$(nix build "github:supabase/postgres/${NIX_FLAKE_VERSION}#psql_${PGVERSION}/bin" --no-link --print-out-paths --extra-experimental-features nix-command --extra-experimental-features flakes) + + # Determine system architecture + ARCH=$(uname -m) + if [ "$ARCH" = "aarch64" ]; then + SYSTEM="aarch64-linux" + elif [ "$ARCH" = "x86_64" ]; then + SYSTEM="x86_64-linux" + else + echo "ERROR: Unsupported architecture: $ARCH" + exit 1 + fi + + # Fetch store path from catalog (avoids expensive nix eval - prevents OOM on small instances) + CATALOG_URL="https://supabase-public-artifacts-bucket.s3.amazonaws.com/nix-catalog/${NIX_FLAKE_VERSION}.json" + echo "Fetching catalog from: $CATALOG_URL" + + if ! CATALOG_RESPONSE=$(curl -sf "$CATALOG_URL"); then + echo "ERROR: Failed to fetch catalog from $CATALOG_URL" + exit 1 + fi + + STORE_PATH=$(echo "$CATALOG_RESPONSE" | jq -r ".\"${SYSTEM}\".\"psql_${PGVERSION}/bin\"") + + if [ -z "$STORE_PATH" ] || [ "$STORE_PATH" = "null" ]; then + echo "ERROR: Could not find store path in catalog for ${SYSTEM}.psql_${PGVERSION}/bin" + echo "Catalog contents:" + echo "$CATALOG_RESPONSE" | jq . + exit 1 + fi + + echo "Store path: $STORE_PATH" + + # Realize from binary cache (no nix evaluation needed!) + nix-store -r "$STORE_PATH" + + PG_UPGRADE_BIN_DIR="$STORE_PATH" PGSHARENEW="$PG_UPGRADE_BIN_DIR/share/postgresql" fi From 08705bc2c52aefc477c184cc327da058b7640f39 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 5 Jan 2026 15:49:12 -0500 Subject: [PATCH 2/5] fix: catalog entries hash plus version --- .github/workflows/ami-release-nix.yml | 31 ++++++------------- .../pg_upgrade_scripts/initiate.sh | 7 +++-- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 507d0e03d..01eb9889f 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -144,37 +144,26 @@ jobs: GIT_SHA="${{ github.sha }}" PG_VERSION="${{ matrix.postgres_version }}" SYSTEM="aarch64-linux" - ATTR="psql_${PG_VERSION}/bin" # Get store path for this build STORE_PATH=$(nix eval --raw ".#psql_${PG_VERSION}/bin.outPath") - # Catalog keyed by git SHA (what the upgrade script has access to) - CATALOG_URL="https://supabase-public-artifacts-bucket.s3.amazonaws.com/nix-catalog/${GIT_SHA}.json" - CATALOG_S3="s3://supabase-public-artifacts-bucket/nix-catalog/${GIT_SHA}.json" + # Each postgres version gets its own catalog file (no race conditions) + CATALOG_S3="s3://supabase-public-artifacts-bucket/nix-catalog/${GIT_SHA}-psql_${PG_VERSION}.json" - # Download existing catalog or create new - if curl -sf "$CATALOG_URL" -o /tmp/catalog.json; then - echo "Updating existing catalog for ${GIT_SHA}" - else - echo '{}' > /tmp/catalog.json - echo "Creating new catalog for ${GIT_SHA}" - fi - - # Add metadata (version and git_sha) and entry for this system/attr - jq --arg ver "$VERSION" \ + # Create catalog JSON for this version + jq -n \ + --arg ver "$VERSION" \ --arg sha "$GIT_SHA" \ --arg sys "$SYSTEM" \ - --arg attr "$ATTR" \ --arg path "$STORE_PATH" \ - '.version = $ver | .git_sha = $sha | if .[$sys] == null then .[$sys] = {} else . end | .[$sys][$attr] = $path' \ - /tmp/catalog.json > /tmp/catalog-updated.json + '{version: $ver, git_sha: $sha, ($sys): $path}' > /tmp/catalog.json - echo "Catalog entry: ${SYSTEM}.${ATTR} = ${STORE_PATH}" - cat /tmp/catalog-updated.json + echo "Catalog for psql_${PG_VERSION}:" + cat /tmp/catalog.json - # Upload updated catalog - aws s3 cp /tmp/catalog-updated.json "$CATALOG_S3" \ + # Upload catalog + aws s3 cp /tmp/catalog.json "$CATALOG_S3" \ --content-type "application/json" echo "Catalog uploaded to ${CATALOG_S3}" diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index ac8798f53..a1616a447 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -318,7 +318,8 @@ EXTRA_NIX_CONF fi # Fetch store path from catalog (avoids expensive nix eval - prevents OOM on small instances) - CATALOG_URL="https://supabase-public-artifacts-bucket.s3.amazonaws.com/nix-catalog/${NIX_FLAKE_VERSION}.json" + # Each postgres version has its own catalog file: {git_sha}-psql_{version}.json + CATALOG_URL="https://supabase-public-artifacts-bucket.s3.amazonaws.com/nix-catalog/${NIX_FLAKE_VERSION}-psql_${PGVERSION}.json" echo "Fetching catalog from: $CATALOG_URL" if ! CATALOG_RESPONSE=$(curl -sf "$CATALOG_URL"); then @@ -326,10 +327,10 @@ EXTRA_NIX_CONF exit 1 fi - STORE_PATH=$(echo "$CATALOG_RESPONSE" | jq -r ".\"${SYSTEM}\".\"psql_${PGVERSION}/bin\"") + STORE_PATH=$(echo "$CATALOG_RESPONSE" | jq -r ".\"${SYSTEM}\"") if [ -z "$STORE_PATH" ] || [ "$STORE_PATH" = "null" ]; then - echo "ERROR: Could not find store path in catalog for ${SYSTEM}.psql_${PGVERSION}/bin" + echo "ERROR: Could not find store path in catalog for ${SYSTEM}" echo "Catalog contents:" echo "$CATALOG_RESPONSE" | jq . exit 1 From 01b77e0d881bfe0b63d2af2276270e46b59eb091 Mon Sep 17 00:00:00 2001 From: Tom Ashley Date: Tue, 6 Jan 2026 18:56:54 +0000 Subject: [PATCH 3/5] chore: use OIDC to auth shared services bucket (#1997) --- .github/workflows/ami-release-nix.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 01eb9889f..76c56cd9c 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -138,6 +138,22 @@ jobs: aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz + - name: GitHub OIDC Auth + uses: aws-actions/configure-aws-credentials@v4.1.0 + with: + aws-region: ap-southeast-1 + role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role + role-session-name: shared-services-jump + + - name: Assume destination role + uses: aws-actions/configure-aws-credentials@v4.1.0 + with: + aws-region: ap-southeast-1 + role-to-assume: arn:aws:iam::279559813984:role/supabase-nix-catalog-artifacts-role-6387512 + role-skip-session-tagging: true + role-session-name: upload-assets + role-chaining: true + - name: Update nix store path catalog run: | VERSION="${{ steps.process_release_version.outputs.version }}" @@ -149,7 +165,7 @@ jobs: STORE_PATH=$(nix eval --raw ".#psql_${PG_VERSION}/bin.outPath") # Each postgres version gets its own catalog file (no race conditions) - CATALOG_S3="s3://supabase-public-artifacts-bucket/nix-catalog/${GIT_SHA}-psql_${PG_VERSION}.json" + CATALOG_S3="s3://${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}/nix-catalog/${GIT_SHA}-psql_${PG_VERSION}.json" # Create catalog JSON for this version jq -n \ From aebc9f72d3bfd17e5bb5d9aafca16f6654f52cd7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 6 Jan 2026 14:38:10 -0500 Subject: [PATCH 4/5] fix: use aws cli to access catalog --- .../pg_upgrade_scripts/initiate.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index a1616a447..3b163695e 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -319,20 +319,21 @@ EXTRA_NIX_CONF # Fetch store path from catalog (avoids expensive nix eval - prevents OOM on small instances) # Each postgres version has its own catalog file: {git_sha}-psql_{version}.json - CATALOG_URL="https://supabase-public-artifacts-bucket.s3.amazonaws.com/nix-catalog/${NIX_FLAKE_VERSION}-psql_${PGVERSION}.json" - echo "Fetching catalog from: $CATALOG_URL" + CATALOG_S3="s3://supabase-internal-artifacts/nix-catalog/${NIX_FLAKE_VERSION}-psql_${PGVERSION}.json" + CATALOG_LOCAL="/tmp/nix-catalog-${NIX_FLAKE_VERSION}-psql_${PGVERSION}.json" + echo "Fetching catalog from: $CATALOG_S3" - if ! CATALOG_RESPONSE=$(curl -sf "$CATALOG_URL"); then - echo "ERROR: Failed to fetch catalog from $CATALOG_URL" + if ! aws s3 cp "$CATALOG_S3" "$CATALOG_LOCAL" --region ap-southeast-1; then + echo "ERROR: Failed to fetch catalog from $CATALOG_S3" exit 1 fi - STORE_PATH=$(echo "$CATALOG_RESPONSE" | jq -r ".\"${SYSTEM}\"") + STORE_PATH=$(jq -r ".\"${SYSTEM}\"" "$CATALOG_LOCAL") if [ -z "$STORE_PATH" ] || [ "$STORE_PATH" = "null" ]; then echo "ERROR: Could not find store path in catalog for ${SYSTEM}" echo "Catalog contents:" - echo "$CATALOG_RESPONSE" | jq . + jq . "$CATALOG_LOCAL" exit 1 fi From d235f25233b93f8e55aa17b4421f418126aaac46 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 Jan 2026 20:23:53 -0500 Subject: [PATCH 5/5] chore: bump to release --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 2ee674f65..addd73909 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,9 +10,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.6.0.027-orioledb" - postgres17: "17.6.1.070" - postgres15: "15.14.1.070" + postgresorioledb-17: "17.6.0.028-orioledb" + postgres17: "17.6.1.071" + postgres15: "15.14.1.071" # Non Postgres Extensions pgbouncer_release: 1.19.0