feat: add stage 1 AMI build caching based on input hash (#1954)

* feat(nix): add stage 1 AMI build caching based on input hash

Implement content-based caching for stage 1 AMI builds by computing a hash from all input sources.
Then the build-ami script checks for existing AMIs with matching input hash before building.

* refactor(ci): extract common AMI build logic into reusable action and add actionlint

Consolidate duplicate AMI build steps across workflows into a shared
composite action. Also introduces actionlint configuration for GitHub
Actions validation for the modified workflows.

* fix: always set AWS region for AWS-related actions

* fix: nix step parameters

* fix: declare different ami name prefix for test and release builds

In order to differentiate AMIs built for testing and those built for
release, we need different AMI name. This commit introduces a new input
parameter `ami_name_prefix`. For test builds, we set this prefix to
include the GitHub run ID, ensuring uniqueness across test builds. For
release builds, we use a static prefix "supabase-postgres".

* chore: remove unused force-deregister variable from stage 2 AMI build

The force-deregister variable is not used in the stage2-nix-psql.pkr.hcl

* chore: remove unused packer variables

* chore: display published AMI names in GitHub Actions logs

* chore(ci): use the same matrix title format as testinfra-ami-build workflow

We changed the version extraction in `testinfra-ami-build.yml` to avoid
the quotes around the version numbers. This commit makes the same change
in `test.yml` to keep things consistent.

Author: jfroche

.github/actionlint.yaml

@@ -3,3 +3,7 @@ self-hosted-runner:
     - aarch64-darwin
     - aarch64-linux
     - blacksmith-32vcpu-ubuntu-2404
+    - blacksmith-2vcpu-ubuntu-2404
+    - blacksmith-2vcpu-ubuntu-2404-arm
+    - blacksmith-4vcpu-ubuntu-2404
+    - large-linux-arm

.github/actions/build-ami/action.yml

@@ -0,0 +1,87 @@
+name: Build AMI
+description: Build both stage 1 and stage 2 AMIs
+
+inputs:
+  postgres_version:
+    description: 'PostgreSQL major version (e.g., 15)'
+    required: true
+  region:
+    description: 'AWS region'
+    required: true
+  ami_regions:
+    description: 'AMI regions as JSON array (e.g., ["us-east-1"])'
+    required: true
+  git_sha:
+    description: 'Git SHA for this build'
+    required: true
+  ami_name_prefix:
+    description: 'Prefix for the AMI name'
+    required: false
+    default: 'supabase-postgres'
+
+outputs:
+  stage2_ami_id:
+    description: 'The AMI ID of the stage 2 build'
+    value: ${{ steps.build-stage2.outputs.stage2_ami_id }}
+  postgres_release_version:
+    description: 'The PostgreSQL release version'
+    value: ${{ steps.generate-vars.outputs.version }}
+  execution_id:
+    description: 'The execution ID for this build'
+    value: ${{ steps.set-execution-id.outputs.execution_id }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set execution ID
+      id: set-execution-id
+      shell: bash
+      run: |
+        EXECUTION_ID="${{ github.run_id }}-${{ inputs.postgres_version }}"
+        echo "EXECUTION_ID=$EXECUTION_ID" >> $GITHUB_ENV
+        echo "execution_id=$EXECUTION_ID" >> $GITHUB_OUTPUT
+
+    - name: Generate common-nix.vars.pkr.hcl
+      id: generate-vars
+      shell: bash
+      run: |
+        PG_VERSION=$(nix run nixpkgs#yq -- -r '.postgres_release["postgres${{ inputs.postgres_version }}"]' ansible/vars.yml)
+        echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
+        echo "" >> common-nix.vars.pkr.hcl
+        git add -f common-nix.vars.pkr.hcl
+        echo "version=$PG_VERSION" >> $GITHUB_OUTPUT
+
+    - name: Build AMI stage 1
+      shell: bash
+      env:
+        POSTGRES_MAJOR_VERSION: ${{ inputs.postgres_version }}
+        POSTGRES_VERSION: ${{ steps.generate-vars.outputs.version }}
+        AWS_MAX_ATTEMPTS: 10
+        AWS_RETRY_MODE: adaptive
+        AWS_REGION: ${{ inputs.region }}
+      run: |
+        nix run .#build-ami -- stage1 \
+          -var "git-head-version=${{ inputs.git_sha }}" \
+          -var "packer-execution-id=${{ env.EXECUTION_ID }}" \
+          -var "ansible_arguments=-e postgresql_major=${{ inputs.postgres_version }}" \
+          -var 'ami_regions=${{ inputs.ami_regions }}' \
+          amazon-arm64-nix.pkr.hcl
+
+    - name: Build AMI stage 2
+      id: build-stage2
+      shell: bash
+      env:
+        POSTGRES_MAJOR_VERSION: ${{ inputs.postgres_version }}
+        POSTGRES_VERSION: ${{ steps.generate-vars.outputs.version }}
+        PACKER_EXECUTION_ID: ${{ env.EXECUTION_ID }}
+        AWS_MAX_ATTEMPTS: 10
+        AWS_RETRY_MODE: adaptive
+        AWS_REGION: ${{ inputs.region }}
+      run: |
+        nix run .#build-ami -- stage2 \
+          -var "git-head-version=${{ inputs.git_sha }}" \
+          -var "packer-execution-id=${{ env.EXECUTION_ID }}" \
+          -var "postgres_major_version=${{ inputs.postgres_version }}" \
+          -var "ami_name=${{ inputs.ami_name_prefix }}" \
+          -var "git_sha=${{ inputs.git_sha }}" \
+          stage2-nix-psql.pkr.hcl

.github/actions/nix-install-ephemeral/action.yml

@@ -5,6 +5,10 @@ inputs:
     description: 'Whether to push build outputs to the Nix binary cache'
     required: false
     default: 'false'
+  aws-region:
+    description: 'AWS region for the Nix binary cache S3 bucket'
+    required: false
+    default: 'us-east-1'
 runs:
   using: 'composite'
   steps:
@@ -13,7 +17,7 @@ runs:
       if: ${{ inputs.push-to-cache == 'true' }}
       with:
         role-to-assume: ${{ env.DEV_AWS_ROLE }}
-        aws-region: "us-east-1"
+        aws-region: ${{ inputs.aws-region }}
         output-credentials: true
         role-duration-seconds: 7200
     - name: Setup AWS credentials for Nix

.github/workflows/ami-release-nix-single.yml

@@ -27,6 +27,7 @@ jobs:
         uses: supabase/postgres/.github/actions/shared-checkout@HEAD
         with:
           ref: ${{ github.event.inputs.branch }}
+
       - name: aws-creds
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -38,56 +39,35 @@ jobs:
       - name: Get current branch SHA
         id: get_sha
         run: |
-          echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+          echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
 
       - name: Install nix
-        uses: cachix/install-nix-action@v27
+        uses: ./.github/actions/nix-install-ephemeral
         with:
-          install_url: https://releases.nixos.org/nix/nix-2.29.1/install
-          extra_nix_config: |
-            substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
-            trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-
-      - name: Set PostgreSQL version environment variable
-        run: |
-          echo "POSTGRES_MAJOR_VERSION=${{ github.event.inputs.postgres_version }}" >> $GITHUB_ENV
-          echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> $GITHUB_ENV
-
-      - name: Generate common-nix.vars.pkr.hcl
-        run: |
-          PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ env.POSTGRES_MAJOR_VERSION }}'"]' ansible/vars.yml)
-          PG_VERSION=$(echo "$PG_VERSION" | tr -d '"')  # Remove any surrounding quotes
-          echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
-          # Ensure there's a newline at the end of the file
-          echo "" >> common-nix.vars.pkr.hcl
-
-      - name: Build AMI stage 1
+          push-to-cache: 'true'
         env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{ steps.get_sha.outputs.sha }}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}"  amazon-arm64-nix.pkr.hcl
+          DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
+          NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
 
-      - name: Build AMI stage 2
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{ steps.get_sha.outputs.sha }}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl
-          POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
+      - name: Build AMI
+        id: build-ami
+        uses: ./.github/actions/build-ami
+        with:
+          postgres_version: ${{ github.event.inputs.postgres_version }}
+          region: us-east-1
+          ami_regions: '["us-east-1"]'
+          git_sha: ${{ steps.get_sha.outputs.sha }}
 
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          VERSION="${{ steps.build-ami.outputs.postgres_release_version }}"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Create nix flake revision tarball
         run: |
           GIT_SHA=${{ steps.get_sha.outputs.sha }}
-          MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
+          MAJOR_VERSION=${{ github.event.inputs.postgres_version }}
 
           mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}"
           echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version"
@@ -105,7 +85,7 @@ jobs:
           ansible-playbook -i localhost \
             -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
             -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
-            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
+            -e "postgres_major_version=${{ github.event.inputs.postgres_version }}" \
             manifest-playbook.yml
 
       - name: Upload nix flake revision to s3 staging
@@ -126,7 +106,7 @@ jobs:
           ansible-playbook -i localhost \
             -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
             -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
-            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
+            -e "postgres_major_version=${{ github.event.inputs.postgres_version }}" \
             manifest-playbook.yml
     
       - name: Upload nix flake revision to s3 prod
@@ -155,10 +135,12 @@ jobs:
       - name: Cleanup resources after build
         if: ${{ always() }}
         run: |
+          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
           aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
 
       - name: Cleanup resources on build cancellation
         if: ${{ cancelled() }}
         run: |
+          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
           aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids 
 

.github/workflows/ami-release-nix.yml

@@ -25,18 +25,13 @@ jobs:
         uses: supabase/postgres/.github/actions/shared-checkout@HEAD
 
       - name: Install nix
-        uses: cachix/install-nix-action@v27
-        with:
-          install_url: https://releases.nixos.org/nix/nix-2.29.1/install
-          extra_nix_config: |
-            substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
-            trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+        uses: ./.github/actions/nix-install-ephemeral
 
       - name: Set PostgreSQL versions
         id: set-versions
         run: |
-          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
-          echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
+          VERSIONS=$(nix run nixpkgs#yq -- -r '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
+          echo "postgres_versions=$VERSIONS" >> "$GITHUB_OUTPUT"
 
   build:
     needs: prepare
@@ -51,6 +46,7 @@ jobs:
     steps:
       - name: Checkout Repo
         uses: supabase/postgres/.github/actions/shared-checkout@HEAD
+
       - name: aws-creds
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -60,12 +56,12 @@ jobs:
           role-duration-seconds: 7200
 
       - name: Install nix
-        uses: cachix/install-nix-action@v27
+        uses: ./.github/actions/nix-install-ephemeral
         with:
-          install_url: https://releases.nixos.org/nix/nix-2.29.1/install
-          extra_nix_config: |
-            substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
-            trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+          push-to-cache: 'true'
+        env:
+          DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
+          NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
 
       - name: Run checks if triggered manually
         if: ${{ github.event_name == 'workflow_dispatch' }}
@@ -76,47 +72,25 @@ jobs:
             exit 1
           fi
 
-      - name: Set PostgreSQL version environment variable
-        run: |
-          echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV
-          echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> $GITHUB_ENV
-
-      - name: Generate common-nix.vars.pkr.hcl
-        run: |
-          PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
-          PG_VERSION=$(echo "$PG_VERSION" | tr -d '"')  # Remove any surrounding quotes
-          echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
-          # Ensure there's a newline at the end of the file
-          echo "" >> common-nix.vars.pkr.hcl
-
-      - name: Build AMI stage 1
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{github.sha}}
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          # why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined?
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' amazon-arm64-nix.pkr.hcl
-
-      - name: Build AMI stage 2
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{github.sha}}
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl
-          POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' stage2-nix-psql.pkr.hcl
+      - name: Build AMI
+        id: build-ami
+        uses: ./.github/actions/build-ami
+        with:
+          postgres_version: ${{ matrix.postgres_version }}
+          region: us-east-1
+          ami_regions: '["us-east-1"]'
+          git_sha: ${{ github.sha }}
 
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          VERSION="${{ steps.build-ami.outputs.postgres_release_version }}"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Create nix flake revision tarball
         run: |
           GIT_SHA=${{github.sha}}
-          MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
+          MAJOR_VERSION=${{ matrix.postgres_version }}
 
           mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}"
           echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version"
@@ -134,7 +108,7 @@ jobs:
           ansible-playbook -i localhost \
             -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
             -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
-            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
+            -e "postgres_major_version=${{ matrix.postgres_version }}" \
             manifest-playbook.yml
 
       - name: Upload nix flake revision to s3 staging
@@ -155,9 +129,9 @@ jobs:
           ansible-playbook -i localhost \
             -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
             -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
-            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
+            -e "postgres_major_version=${{ matrix.postgres_version }}" \
             manifest-playbook.yml
-    
+
       - name: Upload nix flake revision to s3 prod
         run: |
           aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
@@ -184,9 +158,11 @@ jobs:
       - name: Cleanup resources after build
         if: ${{ always() }}
         run: |
+          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
           aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
 
       - name: Cleanup resources on build cancellation
         if: ${{ cancelled() }}
         run: |
+          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
           aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids

.github/workflows/nix-build.yml

@@ -199,6 +199,7 @@ jobs:
     uses: ./.github/workflows/testinfra-ami-build.yml
     secrets:
       DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
+      NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
 
   run-tests:
     needs: [nix-eval, nix-build-packages-aarch64-linux, nix-build-checks-aarch64-linux, nix-build-packages-aarch64-darwin, nix-build-checks-aarch64-darwin, nix-build-packages-x86_64-linux, nix-build-checks-x86_64-linux]

.github/workflows/test.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Set PostgreSQL versions
         id: set-versions
         run: |
-          VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c "split(\"\n\")[:-1]")
+          VERSIONS=$(nix run nixpkgs#yq -- -r '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c "split(\"\n\")[:-1]")
           echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
   build:
     needs: prepare