ROX-29003: Add Operator-index upgrade test

.github/workflows/auto-operator-index-upgrade-test.yml

@@ -0,0 +1,139 @@
+name: "Auto Operator Index Upgrade Test"
+
+on:
+  workflow_dispatch:
+    inputs:
+      acs-version-override:
+        description: 'Override ACS version, skips bundles.yaml diff (e.g. "4.10")'
+        required: false
+        type: string
+      operator-index-image-override:
+        description: 'Override operator-index image, skips Konflux polling (e.g. quay.io/rhacs-eng/stackrox-operator-index:ocp-v4-22-...)'
+        required: false
+        type: string
+  push:
+    branches: [master]
+    paths: ['bundles.yaml']
+  pull_request:
+    paths: ['bundles.yaml']
+
+permissions:
+  contents: read
+
+jobs:
+  derive-inputs:
+    name: Derive operator-index-upgrade test inputs
+    runs-on: ubuntu-latest
+    outputs:
+      acs-versions: ${{ steps.version.outputs.acs-versions }}
+      operator-index-image: ${{ steps.image.outputs.operator-index-image }}
+      ocp-version: ${{ steps.image.outputs.ocp-version }}
+      should-run: ${{ steps.version.outputs.should-run }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Determine ACS version
+        id: version
+        env:
+          ACS_VERSION_OVERRIDE: ${{ inputs.acs-version-override }}
+          OPERATOR_INDEX_IMAGE_OVERRIDE: ${{ inputs.operator-index-image-override }}
+          BASE_REF: ${{ github.base_ref }}
+          EVENT_NAME: ${{ github.event_name }}
+        run: |
+          # Image override without version override is ambiguous, thus fail fast.
+          if [ -n "$OPERATOR_INDEX_IMAGE_OVERRIDE" ] && [ -z "$ACS_VERSION_OVERRIDE" ]; then
+            echo "::error::operator-index-image-override requires acs-version-override to also be set"
+            exit 1
+          fi
+
+          if [ -n "$ACS_VERSION_OVERRIDE" ]; then
+            echo "Version override provided, skipping bundles.yaml diff: $ACS_VERSION_OVERRIDE"
+            echo "acs-versions=[\"$ACS_VERSION_OVERRIDE\"]" >> "$GITHUB_OUTPUT"
+            echo "should-run=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          if [ "$EVENT_NAME" = "pull_request" ]; then
+            DIFF_BASE="origin/$BASE_REF"
+          else
+            # For runs on master branch, diff the previous commit.
+            DIFF_BASE="HEAD~1"
+          fi
+
+          # Lines starting with '+' are additions in the diff; anchor to 'version:' to skip 'oldest_supported_version:' etc.
+          NEW_VERSIONS=$(git diff "$DIFF_BASE" -- bundles.yaml \
+            | grep '^+[[:space:]]*version:' \
+            | sed 's/.*version:[[:space:]]*//' \
+            || true)
+
+          if [ -z "$NEW_VERSIONS" ]; then
+            echo "No new versions found in bundles.yaml diff, skipping operator-index-upgrade test"
+            echo "should-run=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          # Build version JSON array for the matrix.
+          MINORS_JSON=$(echo "$NEW_VERSIONS" \
+            | sed 's/^\([0-9]*\.[0-9]*\).*/\1/' \
+            | jq -Rc '[.,inputs]')
+          echo "New versions added: $(echo "$NEW_VERSIONS" | tr '\n' ' '), testing minors: $MINORS_JSON"
+          echo "acs-versions=$MINORS_JSON" >> "$GITHUB_OUTPUT"
+          echo "should-run=true" >> "$GITHUB_OUTPUT"
+
+      - name: Determine operator-index image
+        id: image
+        if: steps.version.outputs.should-run == 'true'
+        env:
+          OPERATOR_INDEX_IMAGE_OVERRIDE: ${{ inputs.operator-index-image-override }}
+          SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+        run: |
+          if [ -n "$OPERATOR_INDEX_IMAGE_OVERRIDE" ]; then
+            echo "operator-index-image=$OPERATOR_INDEX_IMAGE_OVERRIDE" >> "$GITHUB_OUTPUT"
+            echo "ocp-version=" >> "$GITHUB_OUTPUT"
+            echo "needs-wait=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          # Use the highest OCP version from .tekton/ pipelines to build the image tag.
+          OCP_VERSION=$(ls .tekton/operator-index-ocp-v*-build.yaml \
+            | sed 's/.*operator-index-ocp-\(v[0-9]*-[0-9]*\)-build\.yaml/\1/' \
+            | sort -V | tail -1)
+          echo "Derived OCP_VERSION: $OCP_VERSION"
+
+          TAG="ocp-${OCP_VERSION}-${SHA}-fast"
+          IMAGE="quay.io/rhacs-eng/stackrox-operator-index:${TAG}"
+          echo "Image: $IMAGE"
+          echo "operator-index-image=$IMAGE" >> "$GITHUB_OUTPUT"
+          # Convert v4-22 → 4.22 for the infra create-cluster args.
+          echo "ocp-version=$(echo "$OCP_VERSION" | sed 's/^v//' | tr '-' '.')" >> "$GITHUB_OUTPUT"
+          echo "needs-wait=true" >> "$GITHUB_OUTPUT"
+
+      - name: Wait for operator-index image
+        if: steps.version.outputs.should-run == 'true' && steps.image.outputs.needs-wait == 'true'
+        uses: stackrox/actions/release/wait-for-image@v1
+        with:
+          image: ${{ steps.image.outputs.operator-index-image }}
+          interval: "60"
+          limit: "7200"
+
+  run-upgrade-tests:
+    name: Run operator-index upgrade tests
+    needs: derive-inputs
+    if: needs.derive-inputs.outputs.should-run == 'true'
+    strategy:
+      matrix:
+        acs-version: ${{ fromJson(needs.derive-inputs.outputs.acs-versions) }}
+      # if any matrix job fails, continue to run the rest of the jobs
+      fail-fast: false
+    uses: ./.github/workflows/operator-index-upgrade-test.yml
+    with:
+      operator-index-image: ${{ needs.derive-inputs.outputs.operator-index-image }}
+      acs-version: ${{ matrix.acs-version }}
+      ocp-version: ${{ needs.derive-inputs.outputs.ocp-version }}
+      cluster-lifespan: 3h
+    secrets:
+      INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}

.github/workflows/operator-index-upgrade-test.yml

@@ -0,0 +1,160 @@
+name: "Operator Index Upgrade Test"
+
+on:
+  workflow_dispatch:
+    inputs:
+      operator-index-image:
+        description: 'ACS Operator index image (e.g., quay.io/rhacs-eng/stackrox-operator-index:ocp-v4-22-...)'
+        required: true
+        type: string
+      acs-version:
+        description: 'ACS minor version under test (e.g. 4.10). Determines channel and Y-2 oldest.'
+        required: true
+        type: string
+      ocp-version:
+        description: 'OCP version for the infra cluster (e.g. 4.22). Derived from operator-index-image tag if not set.'
+        required: false
+        default: ''
+        type: string
+      cluster-lifespan:
+        description: 'Cluster lifespan (e.g., 2h, 4h, 8h)'
+        required: false
+        default: '2h'
+        type: string
+  workflow_call:
+    inputs:
+      operator-index-image:
+        description: 'ACS Operator index image'
+        required: true
+        type: string
+      acs-version:
+        description: 'ACS minor version under test (e.g. 4.10). Determines channel and Y-2 oldest.'
+        required: true
+        type: string
+      ocp-version:
+        description: 'OCP version for the infra cluster (e.g. 4.22). Derived from operator-index-image tag if not set.'
+        required: false
+        default: ''
+        type: string
+      cluster-lifespan:
+        description: 'Cluster lifespan (e.g., 2h, 4h, 8h)'
+        required: false
+        default: '2h'
+        type: string
+    secrets:
+      INFRA_TOKEN:
+        required: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  CLUSTER_NAME: upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
+  GH_TOKEN: ${{ github.token }}
+  GH_NO_UPDATE_NOTIFIER: 1
+
+run-name: >-
+  ${{ format('Operator Index Upgrade Test v{0}', inputs.acs-version) }}
+
+jobs:
+  create-cluster:
+    name: Create OpenShift cluster
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set cluster args
+        id: ocp
+        env:
+          OCP_VERSION: ${{ inputs.ocp-version }}
+        run: |
+          if [ -n "$OCP_VERSION" ]; then
+            echo "infra-args=openshift-version=ocp/stable-${OCP_VERSION}" >> "$GITHUB_OUTPUT"
+          else
+            echo "infra-args=" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create OpenShift cluster
+        uses: stackrox/actions/infra/create-cluster@v1
+        with:
+          token: ${{ secrets.INFRA_TOKEN }}
+          flavor: openshift-4
+          name: ${{ env.CLUSTER_NAME }}
+          lifespan: ${{ inputs.cluster-lifespan || '2h' }}
+          wait: true
+          args: ${{ steps.ocp.outputs.infra-args }}
+
+  run-upgrade-tests:
+    name: Run operator-index upgrade tests
+    needs: create-cluster
+    runs-on: ubuntu-latest
+    env:
+      INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
+    container:
+      image: quay.io/stackrox-io/apollo-ci:stackrox-test-stable
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Install infractl
+        uses: stackrox/actions/infra/install-infractl@v1
+
+      - name: Download cluster kubeconfig
+        run: |
+          infractl artifacts --download-dir=./artifacts "$CLUSTER_NAME"
+          echo "KUBECONFIG=${{ github.workspace }}/artifacts/kubeconfig" >> "$GITHUB_ENV"
+
+      - name: "Test 1: upgrade oldest-supported → provided"
+        env:
+          OPERATOR_INDEX_IMAGE: ${{ inputs.operator-index-image }}
+          ACS_VERSION: ${{ inputs.acs-version }}
+        run: make upgrade-test-oldest
+
+      - name: "Test 2: install provided → optionally upgrade to latest GA"
+        env:
+          OPERATOR_INDEX_IMAGE: ${{ inputs.operator-index-image }}
+          ACS_VERSION: ${{ inputs.acs-version }}
+        run: make upgrade-test-latest
+
+  delete-test-cluster:
+    name: Delete infra test cluster
+    needs: [create-cluster, run-upgrade-tests]
+    runs-on: ubuntu-latest
+    if: always() && needs.create-cluster.result != 'cancelled'
+    env:
+      INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
+    steps:
+      - name: Install infractl
+        uses: stackrox/actions/infra/install-infractl@v1
+
+      - name: Delete cluster
+        run: infractl delete "$CLUSTER_NAME"
+
+  report-status:
+    name: Report test status
+    needs: [create-cluster, run-upgrade-tests, delete-test-cluster]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Write summary
+        run: |
+          cat <<'EOF' >> "$GITHUB_STEP_SUMMARY"
+          # Operator Index Upgrade Test Results
+
+          | Field | Value |
+          |-------|-------|
+          | **Operator image** | `${{ inputs.operator-index-image }}` |
+          | **ACS version** | `${{ inputs.acs-version }}` |
+          | **Cluster** | `${{ env.CLUSTER_NAME }}` |
+          | **Create cluster** | ${{ needs.create-cluster.result }} |
+          | **Upgrade tests** | ${{ needs.run-upgrade-tests.result }} |
+          | **Delete cluster** | ${{ needs.delete-test-cluster.result }} |
+
+          ## Tests run:
+          1. **Test 1** — install oldest supported ACS version, upgrade to the provided operator-index image
+          2. **Test 2** — install provided operator-index image, upgrade to latest GA if provided ACS version is behind
+          EOF

Makefile

@@ -36,6 +36,18 @@ catalog-template.yaml: bundles.yaml $(wildcard $(GENERATE_SCRIPT_FOLDER)/*.go)
 go-test:
 	@$(GO) test -cover -v ./cmd/...
 
+# Operator index upgrade tests — require OPERATOR_INDEX_IMAGE and ACS_VERSION env vars.
+# Example: make upgrade-test-oldest OPERATOR_INDEX_IMAGE=quay.io/rhacs-eng/stackrox-operator-index:... ACS_VERSION=4.11
+.PHONY: upgrade-test-oldest upgrade-test-latest upgrade-test
+
+upgrade-test-oldest:
+	$(GO) test -v -count=1 -run TestUpgradeOldest -timeout 45m ./operator-index-upgrade-test/
+
+upgrade-test-latest:
+	$(GO) test -v -count=1 -run TestUpgradeLatest -timeout 30m ./operator-index-upgrade-test/
+
+upgrade-test: upgrade-test-oldest upgrade-test-latest
+
 $(OPM):
 	mkdir -p "$$(dirname $@)"
 	os_name="$$(uname | tr '[:upper:]' '[:lower:]')"; \

operator-index-upgrade-test/main_test.go

@@ -0,0 +1,13 @@
+package upgradetest
+
+import (
+	"os"
+	"testing"
+)
+
+// TestMain resets any leftover operator state before running the suite.
+// This makes local re-runs on the same cluster safe without manual cleanup.
+func TestMain(m *testing.M) {
+	_ = ResetOperator()
+	os.Exit(m.Run())
+}