From fec343ba988be181580b826c78a0abc620e77d70 Mon Sep 17 00:00:00 2001 From: Prachiti Talgulkar Date: Thu, 6 Aug 2026 22:48:31 +0530 Subject: [PATCH] Add new UT for TestMidRolloutImageChange Signed-off-by: Prachiti Talgulkar --- Makefile | 6 +- test/e2e/bootcnode_test.go | 172 +++++++++++++++++++++++++++++++++++++ test/e2e/e2eutil/env.go | 35 ++++++-- 3 files changed, 206 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 5fec160..ffed521 100644 --- a/Makefile +++ b/Makefile @@ -73,6 +73,7 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= ARTIFACTS=$(ARTIFACTS) \ BINK_NODE_IMAGE_DIGEST=$$(skopeo inspect --tls-verify=false --format '{{.Digest}}' docker://localhost:5000/node:latest) \ BINK_NODE_IMAGE_UPDATE_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update | jq -r '.Digest') \ + BINK_NODE_IMAGE_UPDATE2_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update2 | jq -r '.Digest') \ go test -timeout 20m -count=1 $(if $(V),-v) $(if $(RUN),-run $(RUN)) . ##@ Build @@ -93,10 +94,13 @@ buildimg: ## Build container image. $(CONTAINER_TOOL) build -t $(IMG) . .PHONY: build-update-image -build-update-image: ## Build a derived node image for update testing and push to bink registry. +build-update-image: ## Build derived node images for update testing and push to bink registry. @printf 'FROM localhost:5000/node:latest\nRUN touch /usr/share/update-marker\n' | \ podman build -t localhost:5000/node:update -f - . podman push --tls-verify=false localhost:5000/node:update + @printf 'FROM localhost:5000/node:latest\nRUN touch /usr/share/update-marker-2\n' | \ + podman build -t localhost:5000/node:update2 -f - . + podman push --tls-verify=false localhost:5000/node:update2 ##@ Deployment diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index 7ddf51e..808c6d5 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "os/exec" + "strings" "testing" "time" @@ -347,6 +348,177 @@ func TestTagResolution(t *testing.T) { t.Logf("Node %q is Idle with update image", nodeName) } +// TestMidRolloutImageChange provisions two worker nodes, starts a rollout +// to one update image, then switches the target to a different update image +// while one node is rebooting. It verifies both nodes converge to the final +// image and that the non-rebooting node does not wastefully reboot into the +// first update image. +func TestMidRolloutImageChange(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + + env := e2eutil.New(t) + nodeA := env.AddNode(t) + nodeB := env.AddNode(t) + + ctx := context.Background() + + // Phase 1: Create pool with original image and wait for both nodes Idle. + pool := env.NewPool("mid-rollout", env.NodeImageDigestedPullSpec()) + g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) + + for _, nodeName := range []string{nodeA, nodeB} { + g.Eventually(func(g Gomega) { + var bn bootcv1alpha1.BootcNode + g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed()) + g.Expect(bn.Status.Booted).NotTo(BeNil()) + g.Expect(bn.Status.Conditions).To(ContainElement(And( + HaveField("Type", bootcv1alpha1.NodeIdle), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.NodeReasonIdle), + ))) + }).WithTimeout(3 * time.Minute).Should(Succeed()) + } + + t.Logf("Both nodes are Idle with original image") + + // Phase 2: Record boot count for both nodes before the rollout. + bootCountBefore := make(map[string]string) + for _, nodeName := range []string{nodeA, nodeB} { + bootCountBefore[nodeName] = getBootCount(t, env, ctx, nodeName) + t.Logf("Node %q boot count before: %s", nodeName, bootCountBefore[nodeName]) + } + + // Phase 3: Patch pool to first update image. + updateRef1 := env.NodeImageUpdateDigestedPullSpec() + + modified := pool.DeepCopy() + modified.Spec.Image.Ref = updateRef1 + g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed()) + *pool = *modified + + t.Logf("Patched pool to first update image %s", updateRef1) + + // Phase 4: Wait for any node to reach Rebooting. + var rebootingNode, otherNode string + g.Eventually(func(g Gomega) { + for _, nodeName := range []string{nodeA, nodeB} { + var bn bootcv1alpha1.BootcNode + g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed()) + for _, c := range bn.Status.Conditions { + if c.Type == bootcv1alpha1.NodeIdle && + c.Status == metav1.ConditionFalse && + c.Reason == bootcv1alpha1.NodeReasonRebooting { + rebootingNode = nodeName + return + } + } + } + g.Expect(rebootingNode).NotTo(BeEmpty(), "expected at least one node to be Rebooting") + }).WithTimeout(5 * time.Minute).Should(Succeed()) + + if rebootingNode == nodeA { + otherNode = nodeB + } else { + otherNode = nodeA + } + + t.Logf("Node %q is Rebooting, node %q is the other node", rebootingNode, otherNode) + + // Phase 5: Immediately switch target to second update image. + updateRef2 := env.NodeImageUpdate2DigestedPullSpec() + + modified = pool.DeepCopy() + modified.Spec.Image.Ref = updateRef2 + g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed()) + *pool = *modified + + t.Logf("Switched pool to second update image %s", updateRef2) + + // Phase 6: Wait for both nodes to be Idle with the second update image. + for _, nodeName := range []string{nodeA, nodeB} { + g.Eventually(func(g Gomega) { + var bn bootcv1alpha1.BootcNode + g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed()) + g.Expect(bn.Status.Booted).NotTo(BeNil()) + g.Expect(bn.Status.Booted.ImageDigest).To(Equal(env.NodeImageUpdate2Digest()), + "expected booted digest to match second update image") + g.Expect(bn.Status.Conditions).To(ContainElement(And( + HaveField("Type", bootcv1alpha1.NodeIdle), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.NodeReasonIdle), + ))) + }).WithTimeout(8 * time.Minute).Should(Succeed(), + "expected node %s to reach Idle with second update image", nodeName) + } + + t.Logf("Both nodes are Idle with second update image") + + // Phase 7: Verify the other node (the one that was NOT rebooting when + // we switched images) did not wastefully reboot into the first update + // image. It should have rebooted exactly once (into the second image). + bootCountAfter := getBootCount(t, env, ctx, otherNode) + t.Logf("Node %q boot count after: %s (before: %s)", otherNode, bootCountAfter, bootCountBefore[otherNode]) + + beforeCount := 0 + fmt.Sscanf(bootCountBefore[otherNode], "%d", &beforeCount) + afterCount := 0 + fmt.Sscanf(bootCountAfter, "%d", &afterCount) + + g.Expect(afterCount - beforeCount).To(Equal(1), + "expected other node %s to reboot exactly once (from %d to %d), "+ + "an extra reboot means it wastefully booted into the first update image", + otherNode, beforeCount, afterCount) + + t.Logf("Verified node %q rebooted exactly once (no wasteful reboot into first image)", otherNode) +} + +// getBootCount returns the number of boots on a node by running +// journalctl --list-boots inside the daemon pod via kubectl exec. +func getBootCount(t *testing.T, env *e2eutil.Env, ctx context.Context, nodeName string) string { + t.Helper() + + g := NewWithT(t) + + var daemonPod corev1.Pod + g.Eventually(func(g Gomega) { + var pods corev1.PodList + g.Expect(env.Client.List(ctx, &pods, + client.InNamespace("bootc-operator"), + client.MatchingLabels{ + "app.kubernetes.io/name": "bootc-operator", + "app.kubernetes.io/component": "daemon", + }, + )).To(Succeed()) + var matched []corev1.Pod + for _, p := range pods.Items { + if p.Spec.NodeName == nodeName && p.Status.Phase == corev1.PodRunning { + matched = append(matched, p) + } + } + g.Expect(matched).To(HaveLen(1)) + daemonPod = matched[0] + }).WithTimeout(1 * time.Minute).Should(Succeed()) + + kubeconfigPath := os.Getenv("KUBECONFIG") + cmd := exec.CommandContext(ctx, "kubectl", "--kubeconfig", kubeconfigPath, + "-n", "bootc-operator", "exec", daemonPod.Name, "--", + "nsenter", "-m/proc/1/ns/mnt", "--", "journalctl", "--list-boots") + out, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred(), + fmt.Sprintf("journalctl --list-boots failed: %s", string(out))) + + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + count := 0 + for _, line := range lines { + if strings.TrimSpace(line) != "" { + count++ + } + } + return fmt.Sprintf("%d", count) +} + // TestPauseResume provisions a worker node, starts an update with the // pool paused, verifies the node stages but does not reboot, then resumes // and verifies the update completes. diff --git a/test/e2e/e2eutil/env.go b/test/e2e/e2eutil/env.go index aa55d83..effaf49 100644 --- a/test/e2e/e2eutil/env.go +++ b/test/e2e/e2eutil/env.go @@ -66,6 +66,10 @@ type Env struct { // nodeImageUpdateDigest is the manifest digest of the update image // (e.g. "sha256:def456..."). Empty when not built. nodeImageUpdateDigest string + + // nodeImageUpdate2Digest is the manifest digest of the second update + // image (e.g. "sha256:789abc..."). Used by mid-rollout image change tests. + nodeImageUpdate2Digest string } // New connects to an existing bink cluster and returns an Env ready @@ -96,16 +100,21 @@ func New(t *testing.T) *Env { if nodeImageUpdateDigest == "" { t.Fatal("BINK_NODE_IMAGE_UPDATE_DIGEST must be set") } + nodeImageUpdate2Digest := os.Getenv("BINK_NODE_IMAGE_UPDATE2_DIGEST") + if nodeImageUpdate2Digest == "" { + t.Fatal("BINK_NODE_IMAGE_UPDATE2_DIGEST must be set") + } k8sClient := buildClient(t, kubeconfigPath) env := &Env{ - Client: k8sClient, - clusterName: clusterName, - testID: sanitizeTestName(t.Name()), - nodeImageDigest: nodeImageDigest, - nodeImageRegistry: nodeImageRegistry, - nodeImageUpdateDigest: nodeImageUpdateDigest, + Client: k8sClient, + clusterName: clusterName, + testID: sanitizeTestName(t.Name()), + nodeImageDigest: nodeImageDigest, + nodeImageRegistry: nodeImageRegistry, + nodeImageUpdateDigest: nodeImageUpdateDigest, + nodeImageUpdate2Digest: nodeImageUpdate2Digest, } t.Cleanup(func() { @@ -251,6 +260,20 @@ func (e *Env) NodeImageUpdateDigest() string { return e.nodeImageUpdateDigest } +// NodeImageUpdate2DigestedPullSpec returns the digest-qualified reference for the +// second update image (e.g. "registry.cluster.local:5000/node@sha256:789abc"). +func (e *Env) NodeImageUpdate2DigestedPullSpec() string { + if e.nodeImageRegistry == "" || e.nodeImageUpdate2Digest == "" { + return "" + } + return e.nodeImageRegistry + "@" + e.nodeImageUpdate2Digest +} + +// NodeImageUpdate2Digest returns the manifest digest of the second update image. +func (e *Env) NodeImageUpdate2Digest() string { + return e.nodeImageUpdate2Digest +} + // RetagImage reads the image at srcRef from the localhost registry and // tags it as dstTag. func RetagImage(t *testing.T, srcRef, dstTag string) {