Add binary integration testing support to GitHub Actions workflow

.github/workflows/build.yml

@@ -186,18 +186,9 @@ jobs:
         with:
           dotnet-version: '10.0.x'
        # Install and configure Rust & Cargo Lambda
-      - name: Install and configure Rust & Cargo Lambda
+      - name: Install Rust toolchain and cargo-lambda
         if: ${{ matrix.os == 'ubuntu-latest' }}
-        run: |
-          : install rustup if needed
-          if ! command -v rustup &> /dev/null ; then
-            curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y
-            echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH
-          fi
-          rustup toolchain install stable --profile minimal --no-self-update
-          rustup default stable
-          pip install "cargo-lambda==$CARGO_LAMBDA_VERSION"
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
+        run: bash tests/install-rust.sh
       - name: Init samdev
         run: make init
       - name: uv install setuptools in Python3.12

.github/workflows/integration-tests.yml

@@ -5,9 +5,23 @@ on:
     # Run at 7pm PST (3am UTC next day) Monday-Friday
     # This translates to Tuesday-Saturday 3am UTC
     - cron: '0 3 * * 2-6'
+  release:
+    types: [published, edited]
+    # published → latest-release binary test (full releases only)
+    # edited + prerelease → nightly binary test (nightly pre-release gets edited with new artifacts)
   workflow_dispatch:  # Allow manual triggering
     # NOTE: This workflow can only be manually triggered from develop or main branches
     # The other branch will not pass the OIDC permission check
+    inputs:
+      install_mode:
+        description: '"code" to install from source (make init), "nightly-release" to install SAM CLI nightly binary, "latest-release" to install latest stable release binary'
+        required: false
+        default: 'code'
+        type: choice
+        options:
+          - code
+          - nightly-release
+          - latest-release
 
 permissions:
   id-token: write   # Required for OIDC
@@ -153,20 +167,9 @@ jobs:
         with:
           ruby-version: '3.4.7'
 
-      - name: Install Rust toolchain
+      - name: Install Rust toolchain and cargo-lambda
         if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite)
-        run: |
-          curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL https://sh.rustup.rs | sh -s -- --default-toolchain none -y
-          source $HOME/.cargo/env
-          rustup toolchain install stable --profile minimal --no-self-update
-          rustup default stable
-          rustup target add x86_64-unknown-linux-gnu --toolchain stable
-          rustup target add aarch64-unknown-linux-gnu --toolchain stable
-          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
-
-      - name: Install cargo-lambda
-        if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite)
-        run: pip install cargo-lambda==${{ env.CARGO_LAMBDA_VERSION }}
+        run: bash tests/install-rust.sh
 
       - name: Install Terraform
         if: contains(fromJSON('["terraform-build", "terraform-start-api", "terraform-invoke-start-lambda", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite)
@@ -193,18 +196,52 @@ jobs:
         if: "!contains(fromJSON('[\"build-x86-1\", \"build-x86-2\", \"build-x86-container-1\", \"build-x86-container-2\", \"tier1-finch\"]'), matrix.test_suite)"
         run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 
-      - name: Initialize project
+      - name: Determine test type
         run: |
-          if [[ "${{ matrix.test_suite }}" == "tier1-finch" ]]; then
-            export CONTAINER_RUNTIME=finch
+          echo "CONTAINER_RUNTIME=${{ matrix.test_suite == 'tier1-finch' && 'finch' || 'docker' }}" >> $GITHUB_ENV
+          if [[ "${{ inputs.install_mode }}" == "nightly-release" ]] || \
+             [[ "${{ github.event.action }}" == "edited" && "${{ github.event.release.prerelease }}" == "true" ]]; then
+            echo "TEST_TYPE=nightly-release" >> $GITHUB_ENV
+          elif [[ "${{ inputs.install_mode }}" == "latest-release" ]] || \
+               [[ "${{ github.event.action }}" == "published" && "${{ github.event.release.prerelease }}" != "true" ]]; then
+            echo "TEST_TYPE=latest-release" >> $GITHUB_ENV
+          elif [[ "${{ github.ref }}" == "refs/heads/master" ]]; then
+            echo "TEST_TYPE=master" >> $GITHUB_ENV
+          elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ github.ref }}" == "refs/heads/develop" ]]; then
+            echo "TEST_TYPE=develop" >> $GITHUB_ENV
           else
-            export CONTAINER_RUNTIME=docker
+            echo "TEST_TYPE=other" >> $GITHUB_ENV
           fi
+
+      - name: Initialize integration test
+        if: env.TEST_TYPE == 'develop' || env.TEST_TYPE == 'master' || env.TEST_TYPE == 'other'
+        run: |
           make init
+          echo "SCRIPT_PY=python3.11" >> $GITHUB_ENV
+
+      - name: Initialize nightly release test
+        if: env.TEST_TYPE == 'nightly-release'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          make setup-pytest
+          make init-nightly
+          echo "SAM_CLI_DEV=" >> $GITHUB_ENV
+          echo "SCRIPT_PY=$HOME/pytest/bin/python3" >> $GITHUB_ENV
+
+      - name: Initialize latest release test
+        if: env.TEST_TYPE == 'latest-release'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          make setup-pytest
+          make init-latest-release
+          echo "SAM_CLI_DEV=" >> $GITHUB_ENV
+          echo "SCRIPT_PY=$HOME/pytest/bin/python3" >> $GITHUB_ENV
 
       - name: Get testing resources and credentials
         if: "!contains(fromJSON('[\"build-x86-1\", \"build-x86-2\", \"build-arm64\", \"build-x86-container-1\", \"build-x86-container-2\", \"build-arm64-container-1\", \"build-arm64-container-2\", \"local-invoke\", \"local-start-api\", \"local-start-lambda\"]'), matrix.test_suite)"
-        run: python3.11 tests/setup_testing_resources.py
+        run: $SCRIPT_PY tests/setup_testing_resources.py
 
       - name: Login to Public ECR
         run: |
@@ -398,4 +435,4 @@ jobs:
         if: always()
         env:
           TESTREPORTING_S3: ${{ secrets.TESTREPORTING_S3 }}
-        run: python3.11 tests/reset_testing_resources.py ${{ matrix.test_suite }}
+        run: $SCRIPT_PY tests/reset_testing_resources.py ${{ matrix.test_suite }}

Makefile

@@ -2,7 +2,7 @@
 # environment variable.
 SAM_CLI_TELEMETRY ?= 0
 
-.PHONY: schema
+.PHONY: schema init-nightly init-latest-release setup-pytest
 
 # Initialize environment specifically for Github action tests using uv
 init:
@@ -13,6 +13,21 @@ init:
 		SAM_CLI_DEV=1 pip install -e '.[dev]'; \
 	fi
 
+# Set up a pytest venv with test dependencies
+setup-pytest:
+	python3.11 -m venv $(HOME)/pytest
+	uv pip install --python $(HOME)/pytest/bin/python3 -r requirements/dev.txt -r requirements/base.txt
+	sudo ln -sf $(HOME)/pytest/bin/pytest /usr/local/bin/pytest
+	pytest --version
+
+# Install SAM CLI nightly binary
+init-nightly:
+	bash tests/install-sam-cli-binary.sh sam-cli-nightly
+
+# Install SAM CLI latest release binary
+init-latest-release:
+	bash tests/install-sam-cli-binary.sh
+
 test:
 	# Run unit tests and fail if coverage falls below 94%
 	pytest --cov samcli --cov schema --cov-report term-missing --cov-fail-under 94 tests/unit

tests/install-rust.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Install Rust toolchain and cargo-lambda for SAM CLI integration tests.
+# Usage: ./tests/install-rust.sh [CARGO_LAMBDA_VERSION]
+#   CARGO_LAMBDA_VERSION defaults to env var or "v0.17.1"
+set -euo pipefail
+
+CARGO_LAMBDA_VERSION="${1:-${CARGO_LAMBDA_VERSION:-v0.17.1}}"
+
+# Install rustup if not present
+if ! command -v rustup &> /dev/null; then
+  curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL https://sh.rustup.rs | sh -s -- --default-toolchain none -y
+  source "$HOME/.cargo/env"
+  if [ -n "${GITHUB_PATH:-}" ]; then
+    echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> "$GITHUB_PATH"
+  fi
+fi
+
+rustup toolchain install stable --profile minimal --no-self-update
+rustup default stable
+rustup target add x86_64-unknown-linux-gnu --toolchain stable
+rustup target add aarch64-unknown-linux-gnu --toolchain stable
+
+# Install cargo-lambda and ziglang
+python3.11 -m pip install "cargo-lambda==$CARGO_LAMBDA_VERSION"
+
+# Create a zig wrapper so SAM CLI's cargo-lambda can find it
+printf '#!/bin/bash\nexec python3.11 -m ziglang "$@"\n' | sudo tee /usr/local/bin/zig > /dev/null
+sudo chmod +x /usr/local/bin/zig
+
+rustc -V
+cargo -V
+zig version

tests/install-sam-cli-binary.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+# Install SAM CLI binary from a GitHub release, auto-detecting OS and architecture.
+# Usage: ./test/install-sam-cli-binary.sh [<tag>]
+#   If <tag> is provided (e.g. sam-cli-nightly), downloads from that release.
+#   If omitted, downloads from the latest (non-pre-release) release.
+set -euo pipefail
+
+TAG="${1:-}"
+OS="$(uname -s | tr '[:upper:]' '[:lower:]')"
+ARCH="$(uname -m)"
+
+# Map to asset naming convention
+case "$OS" in
+  linux)
+    case "$ARCH" in
+      x86_64)  ASSET="aws-sam-cli-linux-x86_64.zip" ;;
+      aarch64) ASSET="aws-sam-cli-linux-arm64.zip" ;;
+      *)       echo "Unsupported Linux architecture: $ARCH"; exit 1 ;;
+    esac
+    ;;
+  darwin)
+    case "$ARCH" in
+      x86_64) ASSET="aws-sam-cli-macos-x86_64.pkg" ;;
+      arm64)  ASSET="aws-sam-cli-macos-arm64.pkg" ;;
+      *)      echo "Unsupported macOS architecture: $ARCH"; exit 1 ;;
+    esac
+    ;;
+  *)
+    echo "Unsupported OS: $OS"; exit 1
+    ;;
+esac
+
+echo "Detected OS=$OS ARCH=$ARCH -> downloading $ASSET"
+
+# Download to a temp directory and clean up on exit
+TMPDIR="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR"' EXIT
+
+if [ -n "$TAG" ]; then
+  gh release download "$TAG" --repo aws/aws-sam-cli --pattern "$ASSET" --clobber --dir "$TMPDIR"
+else
+  gh release download --repo aws/aws-sam-cli --pattern "$ASSET" --clobber --dir "$TMPDIR"
+fi
+
+# Install
+case "$ASSET" in
+  *.zip)
+    unzip -o "$TMPDIR/$ASSET" -d "$TMPDIR/sam-installation"
+    sudo "$TMPDIR/sam-installation/install" --update
+    ;;
+  *.pkg)
+    sudo installer -pkg "$TMPDIR/$ASSET" -target /
+    ;;
+esac
+
+# Nightly installs as sam-nightly; rename to sam
+if [ -f /usr/local/bin/sam-nightly ]; then
+  sudo mv /usr/local/bin/sam-nightly /usr/local/bin/sam
+fi
+
+sam --version

tests/reset_testing_resources.py

@@ -57,7 +57,8 @@ def upload_test_reports(test_suite):
         s3_client = boto3.client("s3", config=DEFAULT_BOTO_CONFIG)
 
         date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-        s3_prefix = f"github/{date_str}/{run_id}"
+        test_type = os.environ.get("TEST_TYPE", "unknown")
+        s3_prefix = f"github/{test_type}/{date_str}/{run_id}"
 
         for report_path in reports:
             s3_key = f"{s3_prefix}/{report_path}"