e2e: add mid-rollout image change test

Makefile

@@ -73,6 +73,7 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN=<regex>
 		ARTIFACTS=$(ARTIFACTS) \
 		BINK_NODE_IMAGE_DIGEST=$$(skopeo inspect --tls-verify=false --format '{{.Digest}}' docker://localhost:5000/node:latest) \
 		BINK_NODE_IMAGE_UPDATE_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update | jq -r '.Digest') \
+		BINK_NODE_IMAGE_UPDATE2_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update2 | jq -r '.Digest') \
 		go test -timeout 20m -count=1 $(if $(V),-v) $(if $(RUN),-run $(RUN)) .
 
 ##@ Build
@@ -93,10 +94,13 @@ buildimg: ## Build container image.
 	$(CONTAINER_TOOL) build -t $(IMG) .
 
 .PHONY: build-update-image
-build-update-image: ## Build a derived node image for update testing and push to bink registry.
+build-update-image: ## Build derived node images for update testing and push to bink registry.
 	@printf 'FROM localhost:5000/node:latest\nRUN touch /usr/share/update-marker\n' | \
 		podman build -t localhost:5000/node:update -f - .
 	podman push --tls-verify=false localhost:5000/node:update
+	@printf 'FROM localhost:5000/node:latest\nRUN touch /usr/share/update-marker-2\n' | \
+		podman build -t localhost:5000/node:update2 -f - .
+	podman push --tls-verify=false localhost:5000/node:update2
 
 ##@ Deployment
 

test/e2e/bootcnode_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"strings"
 	"testing"
 	"time"
 
@@ -347,6 +348,177 @@ func TestTagResolution(t *testing.T) {
 	t.Logf("Node %q is Idle with update image", nodeName)
 }
 
+// TestMidRolloutImageChange provisions two worker nodes, starts a rollout
+// to one update image, then switches the target to a different update image
+// while one node is rebooting. It verifies both nodes converge to the final
+// image and that the non-rebooting node does not wastefully reboot into the
+// first update image.
+func TestMidRolloutImageChange(t *testing.T) {
+	g := NewWithT(t)
+	g.SetDefaultEventuallyTimeout(pollTimeout)
+	g.SetDefaultEventuallyPollingInterval(pollInterval)
+
+	env := e2eutil.New(t)
+	nodeA := env.AddNode(t)
+	nodeB := env.AddNode(t)
+
+	ctx := context.Background()
+
+	// Phase 1: Create pool with original image and wait for both nodes Idle.
+	pool := env.NewPool("mid-rollout", env.NodeImageDigestedPullSpec())
+	g.Expect(env.Client.Create(ctx, pool)).To(Succeed())
+
+	for _, nodeName := range []string{nodeA, nodeB} {
+		g.Eventually(func(g Gomega) {
+			var bn bootcv1alpha1.BootcNode
+			g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed())
+			g.Expect(bn.Status.Booted).NotTo(BeNil())
+			g.Expect(bn.Status.Conditions).To(ContainElement(And(
+				HaveField("Type", bootcv1alpha1.NodeIdle),
+				HaveField("Status", metav1.ConditionTrue),
+				HaveField("Reason", bootcv1alpha1.NodeReasonIdle),
+			)))
+		}).WithTimeout(3 * time.Minute).Should(Succeed())
+	}
+
+	t.Logf("Both nodes are Idle with original image")
+
+	// Phase 2: Record boot count for both nodes before the rollout.
+	bootCountBefore := make(map[string]string)
+	for _, nodeName := range []string{nodeA, nodeB} {
+		bootCountBefore[nodeName] = getBootCount(t, env, ctx, nodeName)
+		t.Logf("Node %q boot count before: %s", nodeName, bootCountBefore[nodeName])
+	}
+
+	// Phase 3: Patch pool to first update image.
+	updateRef1 := env.NodeImageUpdateDigestedPullSpec()
+
+	modified := pool.DeepCopy()
+	modified.Spec.Image.Ref = updateRef1
+	g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
+	*pool = *modified
+
+	t.Logf("Patched pool to first update image %s", updateRef1)
+
+	// Phase 4: Wait for any node to reach Rebooting.
+	var rebootingNode, otherNode string
+	g.Eventually(func(g Gomega) {
+		for _, nodeName := range []string{nodeA, nodeB} {
+			var bn bootcv1alpha1.BootcNode
+			g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed())
+			for _, c := range bn.Status.Conditions {
+				if c.Type == bootcv1alpha1.NodeIdle &&
+					c.Status == metav1.ConditionFalse &&
+					c.Reason == bootcv1alpha1.NodeReasonRebooting {
+					rebootingNode = nodeName
+					return
+				}
+			}
+		}
+		g.Expect(rebootingNode).NotTo(BeEmpty(), "expected at least one node to be Rebooting")
+	}).WithTimeout(5 * time.Minute).Should(Succeed())
+
+	if rebootingNode == nodeA {
+		otherNode = nodeB
+	} else {
+		otherNode = nodeA
+	}
+
+	t.Logf("Node %q is Rebooting, node %q is the other node", rebootingNode, otherNode)
+
+	// Phase 5: Immediately switch target to second update image.
+	updateRef2 := env.NodeImageUpdate2DigestedPullSpec()
+
+	modified = pool.DeepCopy()
+	modified.Spec.Image.Ref = updateRef2
+	g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
+	*pool = *modified
+
+	t.Logf("Switched pool to second update image %s", updateRef2)
+
+	// Phase 6: Wait for both nodes to be Idle with the second update image.
+	for _, nodeName := range []string{nodeA, nodeB} {
+		g.Eventually(func(g Gomega) {
+			var bn bootcv1alpha1.BootcNode
+			g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed())
+			g.Expect(bn.Status.Booted).NotTo(BeNil())
+			g.Expect(bn.Status.Booted.ImageDigest).To(Equal(env.NodeImageUpdate2Digest()),
+				"expected booted digest to match second update image")
+			g.Expect(bn.Status.Conditions).To(ContainElement(And(
+				HaveField("Type", bootcv1alpha1.NodeIdle),
+				HaveField("Status", metav1.ConditionTrue),
+				HaveField("Reason", bootcv1alpha1.NodeReasonIdle),
+			)))
+		}).WithTimeout(8 * time.Minute).Should(Succeed(),
+			"expected node %s to reach Idle with second update image", nodeName)
+	}
+
+	t.Logf("Both nodes are Idle with second update image")
+
+	// Phase 7: Verify the other node (the one that was NOT rebooting when
+	// we switched images) did not wastefully reboot into the first update
+	// image. It should have rebooted exactly once (into the second image).
+	bootCountAfter := getBootCount(t, env, ctx, otherNode)
+	t.Logf("Node %q boot count after: %s (before: %s)", otherNode, bootCountAfter, bootCountBefore[otherNode])
+
+	beforeCount := 0
+	fmt.Sscanf(bootCountBefore[otherNode], "%d", &beforeCount)
+	afterCount := 0
+	fmt.Sscanf(bootCountAfter, "%d", &afterCount)
+
+	g.Expect(afterCount - beforeCount).To(Equal(1),
+		"expected other node %s to reboot exactly once (from %d to %d), "+
+			"an extra reboot means it wastefully booted into the first update image",
+		otherNode, beforeCount, afterCount)
+
+	t.Logf("Verified node %q rebooted exactly once (no wasteful reboot into first image)", otherNode)
+}
+
+// getBootCount returns the number of boots on a node by running
+// journalctl --list-boots inside the daemon pod via kubectl exec.
+func getBootCount(t *testing.T, env *e2eutil.Env, ctx context.Context, nodeName string) string {
+	t.Helper()
+
+	g := NewWithT(t)
+
+	var daemonPod corev1.Pod
+	g.Eventually(func(g Gomega) {
+		var pods corev1.PodList
+		g.Expect(env.Client.List(ctx, &pods,
+			client.InNamespace("bootc-operator"),
+			client.MatchingLabels{
+				"app.kubernetes.io/name":      "bootc-operator",
+				"app.kubernetes.io/component": "daemon",
+			},
+		)).To(Succeed())
+		var matched []corev1.Pod
+		for _, p := range pods.Items {
+			if p.Spec.NodeName == nodeName && p.Status.Phase == corev1.PodRunning {
+				matched = append(matched, p)
+			}
+		}
+		g.Expect(matched).To(HaveLen(1))
+		daemonPod = matched[0]
+	}).WithTimeout(1 * time.Minute).Should(Succeed())
+
+	kubeconfigPath := os.Getenv("KUBECONFIG")
+	cmd := exec.CommandContext(ctx, "kubectl", "--kubeconfig", kubeconfigPath,
+		"-n", "bootc-operator", "exec", daemonPod.Name, "--",
+		"nsenter", "-m/proc/1/ns/mnt", "--", "journalctl", "--list-boots")
+	out, err := cmd.CombinedOutput()
+	g.Expect(err).NotTo(HaveOccurred(),
+		fmt.Sprintf("journalctl --list-boots failed: %s", string(out)))
+
+	lines := strings.Split(strings.TrimSpace(string(out)), "\n")
+	count := 0
+	for _, line := range lines {
+		if strings.TrimSpace(line) != "" {
+			count++
+		}
+	}
+	return fmt.Sprintf("%d", count)
+}
+
 // TestPauseResume provisions a worker node, starts an update with the
 // pool paused, verifies the node stages but does not reboot, then resumes
 // and verifies the update completes.

test/e2e/e2eutil/env.go

@@ -66,6 +66,10 @@ type Env struct {
 	// nodeImageUpdateDigest is the manifest digest of the update image
 	// (e.g. "sha256:def456..."). Empty when not built.
 	nodeImageUpdateDigest string
+
+	// nodeImageUpdate2Digest is the manifest digest of the second update
+	// image (e.g. "sha256:789abc..."). Used by mid-rollout image change tests.
+	nodeImageUpdate2Digest string
 }
 
 // New connects to an existing bink cluster and returns an Env ready
@@ -96,16 +100,21 @@ func New(t *testing.T) *Env {
 	if nodeImageUpdateDigest == "" {
 		t.Fatal("BINK_NODE_IMAGE_UPDATE_DIGEST must be set")
 	}
+	nodeImageUpdate2Digest := os.Getenv("BINK_NODE_IMAGE_UPDATE2_DIGEST")
+	if nodeImageUpdate2Digest == "" {
+		t.Fatal("BINK_NODE_IMAGE_UPDATE2_DIGEST must be set")
+	}
 
 	k8sClient := buildClient(t, kubeconfigPath)
 
 	env := &Env{
-		Client:                k8sClient,
-		clusterName:           clusterName,
-		testID:                sanitizeTestName(t.Name()),
-		nodeImageDigest:       nodeImageDigest,
-		nodeImageRegistry:     nodeImageRegistry,
-		nodeImageUpdateDigest: nodeImageUpdateDigest,
+		Client:                 k8sClient,
+		clusterName:            clusterName,
+		testID:                 sanitizeTestName(t.Name()),
+		nodeImageDigest:        nodeImageDigest,
+		nodeImageRegistry:      nodeImageRegistry,
+		nodeImageUpdateDigest:  nodeImageUpdateDigest,
+		nodeImageUpdate2Digest: nodeImageUpdate2Digest,
 	}
 
 	t.Cleanup(func() {
@@ -251,6 +260,20 @@ func (e *Env) NodeImageUpdateDigest() string {
 	return e.nodeImageUpdateDigest
 }
 
+// NodeImageUpdate2DigestedPullSpec returns the digest-qualified reference for the
+// second update image (e.g. "registry.cluster.local:5000/node@sha256:789abc").
+func (e *Env) NodeImageUpdate2DigestedPullSpec() string {
+	if e.nodeImageRegistry == "" || e.nodeImageUpdate2Digest == "" {
+		return ""
+	}
+	return e.nodeImageRegistry + "@" + e.nodeImageUpdate2Digest
+}
+
+// NodeImageUpdate2Digest returns the manifest digest of the second update image.
+func (e *Env) NodeImageUpdate2Digest() string {
+	return e.nodeImageUpdate2Digest
+}
+
 // RetagImage reads the image at srcRef from the localhost registry and
 // tags it as dstTag.
 func RetagImage(t *testing.T, srcRef, dstTag string) {