fix: move audit to end of image build

Author: samrose

ansible/files/cis_baseline_check.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# CIS Baseline Validation Check
+#
+# This script validates that the machine matches the committed baseline
+# specification. It's run during the image build to catch configuration
+# drift before releasing an image.
+#
+# Usage: cis_baseline_check.sh <baseline-file> [flake-path]
+
+set -euo pipefail
+
+BASELINE_FILE="${1:-/tmp/ansible-playbook/audit-specs/baselines/baseline.yml}"
+FLAKE_PATH="${2:-/tmp/ansible-playbook}"
+
+echo "============================================================"
+echo "CIS Baseline Validation"
+echo "============================================================"
+echo ""
+echo "Baseline file: $BASELINE_FILE"
+echo "Flake path: $FLAKE_PATH"
+echo ""
+
+# Check baseline file exists
+if [[ ! -f "$BASELINE_FILE" ]]; then
+    echo "ERROR: Baseline file not found: $BASELINE_FILE"
+    echo ""
+    echo "Make sure the baseline.yml is copied to the build machine."
+    exit 1
+fi
+
+# Source nix environment for the ubuntu user
+# The build runs as root but nix is installed for ubuntu
+if [[ -f /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh ]]; then
+    . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh
+fi
+
+# Install cis-audit from the flake if not already installed
+echo "Installing cis-audit from flake..."
+if ! command -v cis-audit &> /dev/null; then
+    # Install cis-audit package which includes goss
+    nix profile install "${FLAKE_PATH}#cis-audit" --accept-flake-config
+    echo "✓ cis-audit installed"
+else
+    echo "✓ cis-audit already available"
+fi
+
+echo ""
+echo "Running baseline validation..."
+echo "------------------------------------------------------------"
+
+# Run cis-audit with the baseline spec
+# The cis-audit wrapper handles running goss with sudo
+if cis-audit --spec "$BASELINE_FILE" --format documentation; then
+    echo "------------------------------------------------------------"
+    echo ""
+    echo "✓ CIS baseline validation PASSED"
+    echo "  Machine configuration matches the committed baseline."
+    echo ""
+    exit 0
+else
+    EXIT_CODE=$?
+    echo "------------------------------------------------------------"
+    echo ""
+    echo "✗ CIS baseline validation FAILED"
+    echo ""
+    echo "  The machine configuration has drifted from the baseline."
+    echo "  Review the failures above and either:"
+    echo "    1. Fix the configuration to match the baseline, OR"
+    echo "    2. Update the baseline if the change is intentional:"
+    echo "       nix run .#cis-generate-spec -- audit-specs/baselines/baseline.yml"
+    echo ""
+    exit $EXIT_CODE
+fi

ansible/playbook.yml

@@ -220,6 +220,12 @@
         systemctl stop fail2ban.service
       when: stage2_nix
 
+    - name: Run CIS baseline validation
+      become: yes
+      shell: |
+        /bin/bash /tmp/ansible-playbook/ansible/files/cis_baseline_check.sh /tmp/ansible-playbook/audit-specs/baselines/baseline.yml
+      when: stage2_nix
+
     - name: nix collect garbage
       become: yes
       shell: |

testinfra/test_ami_nix.py

@@ -824,79 +824,3 @@ def test_postgrest_read_only_session_attrs(host):
                 print("Warning: Failed to restart PostgreSQL after restoring config")
         else:
             print("Warning: Failed to restore PostgreSQL configuration")
-
-
-def test_cis_baseline_audit(host):
-    """Run CIS baseline audit against the machine and report results.
-
-    This test uploads the current baseline.yml from the repo and uses
-    cis-audit to validate the machine against it. The test reports findings
-    but does not fail the build - it's for visibility into configuration drift.
-    """
-    git_sha = os.environ.get("GITHUB_SHA", "HEAD")
-
-    # Find the baseline file relative to the test file location
-    test_dir = Path(__file__).parent.parent
-    baseline_path = test_dir / "audit-specs" / "baselines" / "baseline.yml"
-
-    if not baseline_path.exists():
-        print(f"\n⚠️  Baseline file not found at {baseline_path}")
-        print("Skipping CIS baseline audit - no baseline file available")
-        pytest.skip("Baseline file not found")
-        return
-
-    print(f"\n{'='*60}")
-    print("CIS BASELINE AUDIT")
-    print(f"{'='*60}")
-    print(f"Baseline file: {baseline_path}")
-
-    # Upload baseline file to the instance
-    remote_baseline_path = "/tmp/baseline.yml"
-    try:
-        upload_file_via_sftp(host["ssh"], str(baseline_path), remote_baseline_path)
-        print(f"✓ Uploaded baseline to {remote_baseline_path}")
-    except Exception as e:
-        print(f"✗ Failed to upload baseline file: {e}")
-        pytest.skip(f"Failed to upload baseline: {e}")
-        return
-
-    # Install cis-audit via nix
-    print("\nInstalling cis-audit tool...")
-    install_cmd = f"nix profile install github:supabase/postgres/{git_sha}#cis-audit --refresh 2>&1"
-    result = run_ssh_command(host["ssh"], install_cmd, timeout=300)
-    if not result["succeeded"]:
-        print(f"Warning: {result['stderr'][:500]}")
-
-    # Run cis-audit with documentation format for readable output
-    print("\nRunning CIS baseline validation...")
-    print(f"{'-'*60}")
-
-    # Use the uploaded baseline file (local path, not bundled)
-    validate_cmd = f"~/.nix-profile/bin/cis-audit --spec {remote_baseline_path} --format documentation 2>&1"
-    result = run_ssh_command(host["ssh"], validate_cmd, timeout=600)
-
-    # Print full output for visibility in GitHub Actions logs
-    print(result["stdout"])
-    if result["stderr"]:
-        print(f"\nStderr:\n{result['stderr']}")
-
-    print(f"{'-'*60}")
-
-    # Also run with tap format to get summary counts
-    validate_tap_cmd = f"~/.nix-profile/bin/cis-audit --spec {remote_baseline_path} --format tap 2>&1 | tail -10"
-    result_tap = run_ssh_command(host["ssh"], validate_tap_cmd, timeout=600)
-
-    print(f"\nSummary:")
-    print(result_tap["stdout"])
-
-    # Clean up
-    run_ssh_command(host["ssh"], f"rm -f {remote_baseline_path}")
-
-    print(f"{'='*60}")
-    print("CIS BASELINE AUDIT COMPLETE")
-    print(f"{'='*60}\n")
-
-    # Note: This test intentionally does not assert/fail on validation results
-    # It's meant to provide visibility into configuration state
-    # To make this test fail on drift, uncomment the following:
-    # assert result["succeeded"], "CIS baseline validation found differences"