diff --git a/.github/workflows/auto-operator-index-upgrade-test.yml b/.github/workflows/auto-operator-index-upgrade-test.yml new file mode 100644 index 00000000..a51306d8 --- /dev/null +++ b/.github/workflows/auto-operator-index-upgrade-test.yml @@ -0,0 +1,139 @@ +name: "Auto Operator Index Upgrade Test" + +on: + workflow_dispatch: + inputs: + acs-version-override: + description: 'Override ACS version, skips bundles.yaml diff (e.g. "4.10")' + required: false + type: string + operator-index-image-override: + description: 'Override operator-index image, skips Konflux polling (e.g. quay.io/rhacs-eng/stackrox-operator-index:ocp-v4-22-...)' + required: false + type: string + push: + branches: [master] + paths: ['bundles.yaml'] + pull_request: + paths: ['bundles.yaml'] + +permissions: + contents: read + +jobs: + derive-inputs: + name: Derive operator-index-upgrade test inputs + runs-on: ubuntu-latest + outputs: + acs-versions: ${{ steps.version.outputs.acs-versions }} + operator-index-image: ${{ steps.image.outputs.operator-index-image }} + ocp-version: ${{ steps.image.outputs.ocp-version }} + should-run: ${{ steps.version.outputs.should-run }} + steps: + - name: Check out code + uses: actions/checkout@v7 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Determine ACS version + id: version + env: + ACS_VERSION_OVERRIDE: ${{ inputs.acs-version-override }} + OPERATOR_INDEX_IMAGE_OVERRIDE: ${{ inputs.operator-index-image-override }} + BASE_REF: ${{ github.base_ref }} + EVENT_NAME: ${{ github.event_name }} + run: | + # Image override without version override is ambiguous, thus fail fast. + if [ -n "$OPERATOR_INDEX_IMAGE_OVERRIDE" ] && [ -z "$ACS_VERSION_OVERRIDE" ]; then + echo "::error::operator-index-image-override requires acs-version-override to also be set" + exit 1 + fi + + if [ -n "$ACS_VERSION_OVERRIDE" ]; then + echo "Version override provided, skipping bundles.yaml diff: $ACS_VERSION_OVERRIDE" + echo "acs-versions=[\"$ACS_VERSION_OVERRIDE\"]" >> "$GITHUB_OUTPUT" + echo "should-run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ "$EVENT_NAME" = "pull_request" ]; then + DIFF_BASE="origin/$BASE_REF" + else + # For runs on master branch, diff the previous commit. + DIFF_BASE="HEAD~1" + fi + + # Lines starting with '+' are additions in the diff; anchor to 'version:' to skip 'oldest_supported_version:' etc. + NEW_VERSIONS=$(git diff "$DIFF_BASE" -- bundles.yaml \ + | grep '^+[[:space:]]*version:' \ + | sed 's/.*version:[[:space:]]*//' \ + || true) + + if [ -z "$NEW_VERSIONS" ]; then + echo "No new versions found in bundles.yaml diff, skipping operator-index-upgrade test" + echo "should-run=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Build version JSON array for the matrix. + MINORS_JSON=$(echo "$NEW_VERSIONS" \ + | sed 's/^\([0-9]*\.[0-9]*\).*/\1/' \ + | jq -Rc '[.,inputs]') + echo "New versions added: $(echo "$NEW_VERSIONS" | tr '\n' ' '), testing minors: $MINORS_JSON" + echo "acs-versions=$MINORS_JSON" >> "$GITHUB_OUTPUT" + echo "should-run=true" >> "$GITHUB_OUTPUT" + + - name: Determine operator-index image + id: image + if: steps.version.outputs.should-run == 'true' + env: + OPERATOR_INDEX_IMAGE_OVERRIDE: ${{ inputs.operator-index-image-override }} + SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + run: | + if [ -n "$OPERATOR_INDEX_IMAGE_OVERRIDE" ]; then + echo "operator-index-image=$OPERATOR_INDEX_IMAGE_OVERRIDE" >> "$GITHUB_OUTPUT" + echo "ocp-version=" >> "$GITHUB_OUTPUT" + echo "needs-wait=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Use the highest OCP version from .tekton/ pipelines to build the image tag. + OCP_VERSION=$(ls .tekton/operator-index-ocp-v*-build.yaml \ + | sed 's/.*operator-index-ocp-\(v[0-9]*-[0-9]*\)-build\.yaml/\1/' \ + | sort -V | tail -1) + echo "Derived OCP_VERSION: $OCP_VERSION" + + TAG="ocp-${OCP_VERSION}-${SHA}-fast" + IMAGE="quay.io/rhacs-eng/stackrox-operator-index:${TAG}" + echo "Image: $IMAGE" + echo "operator-index-image=$IMAGE" >> "$GITHUB_OUTPUT" + # Convert v4-22 → 4.22 for the infra create-cluster args. + echo "ocp-version=$(echo "$OCP_VERSION" | sed 's/^v//' | tr '-' '.')" >> "$GITHUB_OUTPUT" + echo "needs-wait=true" >> "$GITHUB_OUTPUT" + + - name: Wait for operator-index image + if: steps.version.outputs.should-run == 'true' && steps.image.outputs.needs-wait == 'true' + uses: stackrox/actions/release/wait-for-image@v1 + with: + image: ${{ steps.image.outputs.operator-index-image }} + interval: "60" + limit: "7200" + + run-upgrade-tests: + name: Run operator-index upgrade tests + needs: derive-inputs + if: needs.derive-inputs.outputs.should-run == 'true' + strategy: + matrix: + acs-version: ${{ fromJson(needs.derive-inputs.outputs.acs-versions) }} + # if any matrix job fails, continue to run the rest of the jobs + fail-fast: false + uses: ./.github/workflows/operator-index-upgrade-test.yml + with: + operator-index-image: ${{ needs.derive-inputs.outputs.operator-index-image }} + acs-version: ${{ matrix.acs-version }} + ocp-version: ${{ needs.derive-inputs.outputs.ocp-version }} + cluster-lifespan: 3h + secrets: + INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }} diff --git a/.github/workflows/operator-index-upgrade-test.yml b/.github/workflows/operator-index-upgrade-test.yml new file mode 100644 index 00000000..b97fb7b9 --- /dev/null +++ b/.github/workflows/operator-index-upgrade-test.yml @@ -0,0 +1,160 @@ +name: "Operator Index Upgrade Test" + +on: + workflow_dispatch: + inputs: + operator-index-image: + description: 'ACS Operator index image (e.g., quay.io/rhacs-eng/stackrox-operator-index:ocp-v4-22-...)' + required: true + type: string + acs-version: + description: 'ACS minor version under test (e.g. 4.10). Determines channel and Y-2 oldest.' + required: true + type: string + ocp-version: + description: 'OCP version for the infra cluster (e.g. 4.22). Derived from operator-index-image tag if not set.' + required: false + default: '' + type: string + cluster-lifespan: + description: 'Cluster lifespan (e.g., 2h, 4h, 8h)' + required: false + default: '2h' + type: string + workflow_call: + inputs: + operator-index-image: + description: 'ACS Operator index image' + required: true + type: string + acs-version: + description: 'ACS minor version under test (e.g. 4.10). Determines channel and Y-2 oldest.' + required: true + type: string + ocp-version: + description: 'OCP version for the infra cluster (e.g. 4.22). Derived from operator-index-image tag if not set.' + required: false + default: '' + type: string + cluster-lifespan: + description: 'Cluster lifespan (e.g., 2h, 4h, 8h)' + required: false + default: '2h' + type: string + secrets: + INFRA_TOKEN: + required: true + +permissions: + contents: read + +defaults: + run: + shell: bash + +env: + CLUSTER_NAME: upgrade-test-${{ github.run_id }}-${{ github.run_attempt }} + GH_TOKEN: ${{ github.token }} + GH_NO_UPDATE_NOTIFIER: 1 + +run-name: >- + ${{ format('Operator Index Upgrade Test v{0}', inputs.acs-version) }} + +jobs: + create-cluster: + name: Create OpenShift cluster + runs-on: ubuntu-latest + steps: + - name: Set cluster args + id: ocp + env: + OCP_VERSION: ${{ inputs.ocp-version }} + run: | + if [ -n "$OCP_VERSION" ]; then + echo "infra-args=openshift-version=ocp/stable-${OCP_VERSION}" >> "$GITHUB_OUTPUT" + else + echo "infra-args=" >> "$GITHUB_OUTPUT" + fi + + - name: Create OpenShift cluster + uses: stackrox/actions/infra/create-cluster@v1 + with: + token: ${{ secrets.INFRA_TOKEN }} + flavor: openshift-4 + name: ${{ env.CLUSTER_NAME }} + lifespan: ${{ inputs.cluster-lifespan || '2h' }} + wait: true + args: ${{ steps.ocp.outputs.infra-args }} + + run-upgrade-tests: + name: Run operator-index upgrade tests + needs: create-cluster + runs-on: ubuntu-latest + env: + INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }} + container: + image: quay.io/stackrox-io/apollo-ci:stackrox-test-stable + steps: + - name: Check out code + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Install infractl + uses: stackrox/actions/infra/install-infractl@v1 + + - name: Download cluster kubeconfig + run: | + infractl artifacts --download-dir=./artifacts "$CLUSTER_NAME" + echo "KUBECONFIG=${{ github.workspace }}/artifacts/kubeconfig" >> "$GITHUB_ENV" + + - name: "Test 1: upgrade oldest-supported → provided" + env: + OPERATOR_INDEX_IMAGE: ${{ inputs.operator-index-image }} + ACS_VERSION: ${{ inputs.acs-version }} + run: make upgrade-test-oldest + + - name: "Test 2: install provided → optionally upgrade to latest GA" + env: + OPERATOR_INDEX_IMAGE: ${{ inputs.operator-index-image }} + ACS_VERSION: ${{ inputs.acs-version }} + run: make upgrade-test-latest + + delete-test-cluster: + name: Delete infra test cluster + needs: [create-cluster, run-upgrade-tests] + runs-on: ubuntu-latest + if: always() && needs.create-cluster.result != 'cancelled' + env: + INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }} + steps: + - name: Install infractl + uses: stackrox/actions/infra/install-infractl@v1 + + - name: Delete cluster + run: infractl delete "$CLUSTER_NAME" + + report-status: + name: Report test status + needs: [create-cluster, run-upgrade-tests, delete-test-cluster] + runs-on: ubuntu-latest + if: always() + steps: + - name: Write summary + run: | + cat <<'EOF' >> "$GITHUB_STEP_SUMMARY" + # Operator Index Upgrade Test Results + + | Field | Value | + |-------|-------| + | **Operator image** | `${{ inputs.operator-index-image }}` | + | **ACS version** | `${{ inputs.acs-version }}` | + | **Cluster** | `${{ env.CLUSTER_NAME }}` | + | **Create cluster** | ${{ needs.create-cluster.result }} | + | **Upgrade tests** | ${{ needs.run-upgrade-tests.result }} | + | **Delete cluster** | ${{ needs.delete-test-cluster.result }} | + + ## Tests run: + 1. **Test 1** — install oldest supported ACS version, upgrade to the provided operator-index image + 2. **Test 2** — install provided operator-index image, upgrade to latest GA if provided ACS version is behind + EOF diff --git a/Makefile b/Makefile index 4ba1ce0c..209b7951 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,18 @@ catalog-template.yaml: bundles.yaml $(wildcard $(GENERATE_SCRIPT_FOLDER)/*.go) go-test: @$(GO) test -cover -v ./cmd/... +# Operator index upgrade tests — require OPERATOR_INDEX_IMAGE and ACS_VERSION env vars. +# Example: make upgrade-test-oldest OPERATOR_INDEX_IMAGE=quay.io/rhacs-eng/stackrox-operator-index:... ACS_VERSION=4.11 +.PHONY: upgrade-test-oldest upgrade-test-latest upgrade-test + +upgrade-test-oldest: + $(GO) test -v -count=1 -run TestUpgradeOldest -timeout 45m ./operator-index-upgrade-test/ + +upgrade-test-latest: + $(GO) test -v -count=1 -run TestUpgradeLatest -timeout 30m ./operator-index-upgrade-test/ + +upgrade-test: upgrade-test-oldest upgrade-test-latest + $(OPM): mkdir -p "$$(dirname $@)" os_name="$$(uname | tr '[:upper:]' '[:lower:]')"; \ diff --git a/operator-index-upgrade-test/main_test.go b/operator-index-upgrade-test/main_test.go new file mode 100644 index 00000000..dadb010b --- /dev/null +++ b/operator-index-upgrade-test/main_test.go @@ -0,0 +1,13 @@ +package upgradetest + +import ( + "os" + "testing" +) + +// TestMain resets any leftover operator state before running the suite. +// This makes local re-runs on the same cluster safe without manual cleanup. +func TestMain(m *testing.M) { + _ = ResetOperator() + os.Exit(m.Run()) +} diff --git a/operator-index-upgrade-test/olm.go b/operator-index-upgrade-test/olm.go new file mode 100644 index 00000000..6261dcec --- /dev/null +++ b/operator-index-upgrade-test/olm.go @@ -0,0 +1,328 @@ +package upgradetest + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "time" + + yaml "github.com/goccy/go-yaml" +) + +func ocRun(args ...string) error { + cmd := exec.Command("oc", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func ocOutput(args ...string) (string, error) { + var stderr bytes.Buffer + cmd := exec.Command("oc", args...) + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil && stderr.Len() > 0 { + return strings.TrimSpace(string(out)), fmt.Errorf("%w\nstderr: %s", err, strings.TrimSpace(stderr.String())) + } + return strings.TrimSpace(string(out)), err +} + +func ocApply(manifest string) error { + cmd := exec.Command("oc", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(manifest) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +type bundlesYAML struct { + OldestSupportedVersion string `yaml:"oldest_supported_version"` +} + +// ReadOldestSupportedVersion parses oldest_supported_version from bundles.yaml. +func ReadOldestSupportedVersion() (major, minor int, err error) { + data, err := os.ReadFile("../bundles.yaml") + if err != nil { + return 0, 0, fmt.Errorf("reading bundles.yaml: %w", err) + } + var b bundlesYAML + if err := yaml.Unmarshal(data, &b); err != nil { + return 0, 0, fmt.Errorf("parsing bundles.yaml: %w", err) + } + // Strip optional patch component ("4.9.0" → "4.9") before delegating to ParseACSVersion. + parts := strings.SplitN(b.OldestSupportedVersion, ".", 3) + if len(parts) < 2 { + return 0, 0, fmt.Errorf("invalid oldest_supported_version: %s", b.OldestSupportedVersion) + } + return ParseACSVersion(parts[0] + "." + parts[1]) +} + +// ParseACSVersion parses "4.10" into (major=4, minor=10). +func ParseACSVersion(ver string) (major, minor int, err error) { + parts := strings.SplitN(ver, ".", 2) + if len(parts) != 2 { + return 0, 0, fmt.Errorf("ACS_VERSION must be MAJOR.MINOR (e.g. 4.10), got: %q", ver) + } + maj, err := strconv.Atoi(parts[0]) + if err != nil { + return 0, 0, fmt.Errorf("invalid major in %q: %w", ver, err) + } + min, err := strconv.Atoi(parts[1]) + if err != nil { + return 0, 0, fmt.Errorf("invalid minor in %q: %w", ver, err) + } + return maj, min, nil +} + +// GetLatestOfficialMinor returns the highest available minor for the given major +// from the official redhat-operators catalog, retrying until timeout because +// packagemanifests can lag behind the catalog's READY state. +func GetLatestOfficialMinor(major int, timeout time.Duration) (int, error) { + deadline := time.Now().Add(timeout) + prefix := fmt.Sprintf("rhacs-%d.", major) + fmt.Printf(" Looking for latest rhacs-%d.* channel in redhat-operators (timeout: %v)...\n", major, timeout) + for time.Now().Before(deadline) { + out, err := exec.Command("oc", "get", "packagemanifest", + "-n", "openshift-marketplace", + "-l", "catalog=redhat-operators", + "-o", `jsonpath={range .items[?(@.metadata.name=="rhacs-operator")].status.channels[*]}{.name}{"\n"}{end}`, + ).Output() + if err != nil { + fmt.Printf(" packagemanifest query failed: %v, retrying in 10s...\n", err) + time.Sleep(10 * time.Second) + continue + } + maxMinor := -1 + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if !strings.HasPrefix(line, prefix) { + continue + } + n, err := strconv.Atoi(strings.TrimPrefix(line, prefix)) + if err != nil { + continue + } + if n > maxMinor { + maxMinor = n + } + } + if maxMinor >= 0 { + return maxMinor, nil + } + fmt.Printf(" rhacs-%d.* channels not yet in packagemanifest, retrying in 10s...\n", major) + time.Sleep(10 * time.Second) + } + return 0, fmt.Errorf("no rhacs-%d.* channels found in redhat-operators after %v", major, timeout) +} + +// DisableDefaultSources disables all default OperatorHub catalog sources. +func DisableDefaultSources() error { + fmt.Println(" Disabling default OperatorHub sources...") + return ocRun("patch", "OperatorHub", "cluster", + "--type", "json", + "-p", `[{"op":"add","path":"/spec/disableAllDefaultSources","value":true}]`) +} + +// EnableDefaultSources re-enables all default OperatorHub catalog sources. +func EnableDefaultSources() error { + fmt.Println(" Enabling default OperatorHub sources...") + return ocRun("patch", "OperatorHub", "cluster", + "--type", "json", + "-p", `[{"op":"add","path":"/spec/disableAllDefaultSources","value":false}]`) +} + +// ApplyCustomCatalog creates or updates the custom CatalogSource. +func ApplyCustomCatalog(indexImage string) error { + fmt.Printf(" Applying custom CatalogSource (image: %s)...\n", indexImage) + return ocApply(fmt.Sprintf(`apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: my-operator-catalog + namespace: openshift-marketplace +spec: + sourceType: grpc + image: %s + displayName: My Operator Catalog + publisher: Custom`, indexImage)) +} + +// WaitForCatalog polls until the named CatalogSource reaches READY state. +func WaitForCatalog(name, ns string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + fmt.Printf(" Waiting for CatalogSource/%s to be READY (timeout: %v)...\n", name, timeout) + for time.Now().Before(deadline) { + state, _ := ocOutput("get", "catalogsource", name, "-n", ns, + "-o", "jsonpath={.status.connectionState.lastObservedState}") + if state == "READY" { + fmt.Printf(" ✅ CatalogSource/%s is READY\n", name) + return nil + } + fmt.Printf(" state=%s, retrying in 10s...\n", state) + time.Sleep(10 * time.Second) + } + return fmt.Errorf("CatalogSource/%s not READY within %v", name, timeout) +} + +// ApplySubscription creates or updates the rhacs-operator Subscription. +func ApplySubscription(channel, source, sourceNS string) error { + fmt.Printf(" Applying Subscription (channel=%s, source=%s)...\n", channel, source) + return ocApply(fmt.Sprintf(`apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: rhacs-operator + namespace: openshift-operators +spec: + channel: %s + installPlanApproval: Automatic + name: rhacs-operator + source: %s + sourceNamespace: %s`, channel, source, sourceNS)) +} + +// ResolveTargetCSV returns the currentCSV for a channel from the specified catalog, +// retrying until timeout because packagemanifests can lag behind catalog READY. +func ResolveTargetCSV(catalogLabel, channel string, timeout time.Duration) (string, error) { + deadline := time.Now().Add(timeout) + fmt.Printf(" Resolving currentCSV for channel %s from %s...\n", channel, catalogLabel) + jsonpath := fmt.Sprintf( + `{range .items[?(@.metadata.name=="rhacs-operator")].status.channels[?(@.name=="%s")]}{.currentCSV}{end}`, + channel) + for time.Now().Before(deadline) { + csv, _ := ocOutput("get", "packagemanifest", + "-n", "openshift-marketplace", + "-l", "catalog="+catalogLabel, + "-o", "jsonpath="+jsonpath) + if csv != "" { + fmt.Printf(" Target CSV: %s\n", csv) + return csv, nil + } + fmt.Println(" packagemanifest not ready yet, retrying in 10s...") + time.Sleep(10 * time.Second) + } + return "", fmt.Errorf("could not resolve currentCSV for channel %s from %s after %v", + channel, catalogLabel, timeout) +} + +// InstallFromOfficial installs the ACS Operator from redhat-operators at channel rhacs-MAJOR.MINOR. +func InstallFromOfficial(major, minor int) (targetCSV string, err error) { + channel := fmt.Sprintf("rhacs-%d.%d", major, minor) + fmt.Printf(" Installing ACS Operator from redhat-operators, channel %s...\n", channel) + if err := WaitForCatalog("redhat-operators", "openshift-marketplace", 2*time.Minute); err != nil { + return "", err + } + csv, err := ResolveTargetCSV("redhat-operators", channel, 2*time.Minute) + if err != nil { + return "", err + } + if err := ApplySubscription(channel, "redhat-operators", "openshift-marketplace"); err != nil { + return "", err + } + return csv, nil +} + +// applyCustomCatalogAndSubscribe is the shared implementation for InstallFromCustom and UpgradeViaCustom. +func applyCustomCatalogAndSubscribe(indexImage, channel string) (string, error) { + if err := DisableDefaultSources(); err != nil { + return "", err + } + if err := ApplyCustomCatalog(indexImage); err != nil { + return "", err + } + if err := WaitForCatalog("my-operator-catalog", "openshift-marketplace", 3*time.Minute); err != nil { + return "", err + } + csv, err := ResolveTargetCSV("my-operator-catalog", channel, time.Minute) + if err != nil { + return "", err + } + if err := ApplySubscription(channel, "my-operator-catalog", "openshift-marketplace"); err != nil { + return "", err + } + return csv, nil +} + +// InstallFromCustom installs from the custom index on the given channel, disabling default sources. +func InstallFromCustom(indexImage, channel string) (targetCSV string, err error) { + fmt.Printf(" Installing ACS Operator from custom index, channel %s...\n", channel) + return applyCustomCatalogAndSubscribe(indexImage, channel) +} + +// UpgradeViaCustom upgrades by refreshing the custom CatalogSource and updating the subscription. +func UpgradeViaCustom(indexImage, channel string) (targetCSV string, err error) { + fmt.Printf(" Upgrading via custom catalog (channel: %s)...\n", channel) + csv, err := applyCustomCatalogAndSubscribe(indexImage, channel) + if err != nil { + return "", err + } + fmt.Printf(" ✅ Subscription updated — OLM will upgrade to %s\n", csv) + return csv, nil +} + +// UpgradeToLatestOfficial upgrades to the latest GA via redhat-operators. +func UpgradeToLatestOfficial(major int) (targetCSV string, err error) { + if err := EnableDefaultSources(); err != nil { + return "", err + } + if err := WaitForCatalog("redhat-operators", "openshift-marketplace", 3*time.Minute); err != nil { + return "", err + } + latestMinor, err := GetLatestOfficialMinor(major, 3*time.Minute) + if err != nil { + return "", err + } + channel := fmt.Sprintf("rhacs-%d.%d", major, latestMinor) + fmt.Printf(" Upgrading to latest GA channel: %s...\n", channel) + csv, err := ResolveTargetCSV("redhat-operators", channel, 3*time.Minute) + if err != nil { + return "", err + } + if err := ApplySubscription(channel, "redhat-operators", "openshift-marketplace"); err != nil { + return "", err + } + fmt.Printf(" ✅ Subscription updated to %s, target: %s\n", channel, csv) + return csv, nil +} + +// WaitForCSV polls until the target CSV reaches Succeeded phase. +// It correctly handles multi-hop OLM upgrade graphs (e.g. 4.8→4.9→4.10). +func WaitForCSV(targetCSV string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + fmt.Printf(" Waiting for %s to reach Succeeded (timeout: %v)...\n", targetCSV, timeout) + for time.Now().Before(deadline) { + phase, _ := ocOutput("get", "csv", targetCSV, + "-n", "openshift-operators", + "-o", "jsonpath={.status.phase}") + if phase == "Succeeded" { + fmt.Printf(" ✅ %s is Succeeded\n", targetCSV) + return nil + } + if phase == "Failed" { + return fmt.Errorf("%s reached Failed phase — OLM will not recover without a resource change", targetCSV) + } + // Show all in-progress rhacs CSVs so logs show the hop chain. + progress, _ := ocOutput("get", "csv", "-n", "openshift-operators", "--no-headers") + fmt.Printf(" phase=%s, csvs=%s, waiting 15s...\n", phase, + strings.ReplaceAll(progress, "\n", " | ")) + time.Sleep(15 * time.Second) + } + return fmt.Errorf("%s did not reach Succeeded within %v", targetCSV, timeout) +} + +// ResetOperator removes the operator subscription, CSVs, and custom catalog between tests. +func ResetOperator() error { + fmt.Println(" Resetting operator state...") + _ = ocRun("delete", "subscription", "rhacs-operator", + "-n", "openshift-operators", "--ignore-not-found") + out, _ := ocOutput("get", "csv", "-n", "openshift-operators", "--no-headers") + for _, line := range strings.Split(out, "\n") { + if fields := strings.Fields(line); len(fields) > 0 && strings.HasPrefix(fields[0], "rhacs-operator.") { + _ = ocRun("delete", "csv", fields[0], + "-n", "openshift-operators", "--ignore-not-found") + } + } + _ = ocRun("delete", "catalogsource", "my-operator-catalog", + "-n", "openshift-marketplace", "--ignore-not-found") + return EnableDefaultSources() +} diff --git a/operator-index-upgrade-test/upgrade_latest_test.go b/operator-index-upgrade-test/upgrade_latest_test.go new file mode 100644 index 00000000..c92ccf0b --- /dev/null +++ b/operator-index-upgrade-test/upgrade_latest_test.go @@ -0,0 +1,56 @@ +package upgradetest + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestUpgradeLatest tests the install + optional upgrade to latest GA path: +// 1. Installs ACS Operator from OPERATOR_INDEX_IMAGE on the ACS_VERSION channel. +// 2. If ACS_VERSION minor < latest GA minor for the same major: upgrades to latest GA via redhat-operators. +// 3. Verifies each CSV reaches Succeeded. +func TestUpgradeLatest(t *testing.T) { + operatorIndexImage := requireEnv(t, "OPERATOR_INDEX_IMAGE") + acsVersion := requireEnv(t, "ACS_VERSION") + + major, minor, err := ParseACSVersion(acsVersion) + require.NoError(t, err, "parse ACS_VERSION") + channel := fmt.Sprintf("rhacs-%d.%d", major, minor) + + t.Cleanup(func() { _ = ResetOperator() }) + + t.Logf("Image: %s", operatorIndexImage) + t.Logf("Version: %d.%d | Channel: %s", major, minor, channel) + + // Query latest GA minor BEFORE disabling default sources (install_from_custom disables them). + // Use || to degrade gracefully: a catalog timeout just skips the upgrade check. + var latestMinor int + if err := WaitForCatalog("redhat-operators", "openshift-marketplace", 3*time.Minute); err != nil { + t.Logf("Warning: could not wait for redhat-operators: %v — upgrade check will be skipped", err) + } else if lm, err := GetLatestOfficialMinor(major, 3*time.Minute); err != nil { + t.Logf("Warning: could not determine latest GA minor for %d.x: %v — upgrade check will be skipped", major, err) + } else { + latestMinor = lm + t.Logf("Latest GA in redhat-operators: %d.%d", major, latestMinor) + } + + t.Logf("Step 1: Install ACS Operator %s from custom index", channel) + targetCSV, err := InstallFromCustom(operatorIndexImage, channel) + require.NoError(t, err) + require.NoError(t, WaitForCSV(targetCSV, 10*time.Minute)) + + // Upgrade only when the tested major matches and our minor is behind latest. + // Comparing major prevents a spurious upgrade when testing a pre-release (e.g. 5.0 vs 4.12). + if latestMinor > 0 && minor < latestMinor { + t.Logf("Step 2: Upgrade %d.%d → %d.%d (latest GA)", major, minor, major, latestMinor) + targetCSV, err = UpgradeToLatestOfficial(major) + require.NoError(t, err) + require.NoError(t, WaitForCSV(targetCSV, 10*time.Minute)) + } else if latestMinor > 0 { + t.Logf("%d.%d is already at or ahead of the latest GA minor (%d) — no upgrade needed", + major, minor, latestMinor) + } +} diff --git a/operator-index-upgrade-test/upgrade_oldest_test.go b/operator-index-upgrade-test/upgrade_oldest_test.go new file mode 100644 index 00000000..d71731fe --- /dev/null +++ b/operator-index-upgrade-test/upgrade_oldest_test.go @@ -0,0 +1,49 @@ +package upgradetest + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func requireEnv(t *testing.T, name string) string { + t.Helper() + val := os.Getenv(name) + require.NotEmptyf(t, val, "%s env var must be set", name) + return val +} + +// TestUpgradeOldest tests the upgrade path from the oldest supported version to the provided index: +// 1. Installs ACS Operator (oldest_supported_version from bundles.yaml) from official redhat-operators. +// 2. Upgrades to OPERATOR_INDEX_IMAGE on the ACS_VERSION channel. +// 3. Verifies each CSV reaches Succeeded. +func TestUpgradeOldest(t *testing.T) { + operatorIndexImage := requireEnv(t, "OPERATOR_INDEX_IMAGE") + acsVersion := requireEnv(t, "ACS_VERSION") + + major, minor, err := ParseACSVersion(acsVersion) + require.NoError(t, err, "parse ACS_VERSION") + channel := fmt.Sprintf("rhacs-%d.%d", major, minor) + + oldestMajor, oldestMinor, err := ReadOldestSupportedVersion() + require.NoError(t, err, "read oldest_supported_version from bundles.yaml") + + t.Cleanup(func() { _ = ResetOperator() }) + + t.Logf("Image: %s", operatorIndexImage) + t.Logf("Version: %d.%d | Channel: %s", major, minor, channel) + t.Logf("Oldest: %d.%d (from bundles.yaml)", oldestMajor, oldestMinor) + + t.Logf("Step 1: Install ACS Operator %d.%d from redhat-operators", oldestMajor, oldestMinor) + targetCSV, err := InstallFromOfficial(oldestMajor, oldestMinor) + require.NoError(t, err) + require.NoError(t, WaitForCSV(targetCSV, 10*time.Minute)) + + t.Logf("Step 2: Upgrade to %s via custom index", channel) + targetCSV, err = UpgradeViaCustom(operatorIndexImage, channel) + require.NoError(t, err) + require.NoError(t, WaitForCSV(targetCSV, 30*time.Minute)) +}