From bdef9404eca15582beb355be7b99a6623b9f9c54 Mon Sep 17 00:00:00 2001 From: Prachiti Talgulkar Date: Thu, 27 Aug 2026 20:04:07 +0530 Subject: [PATCH] e2e: add controller recovery test Signed-off-by: Prachiti Talgulkar --- .github/workflows/ci.yaml | 2 +- Makefile | 2 +- test/e2e/bootcnode_test.go | 169 ++++++++++++++++++++++++++++++++++++- 3 files changed, 169 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b62d921f..6d8de8f0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -109,7 +109,7 @@ jobs: name: e2e (${{ matrix.kube-minor }}) runs-on: ubuntu-latest needs: [build-bink, build-operator] - timeout-minutes: 45 + timeout-minutes: 50 permissions: contents: read strategy: diff --git a/Makefile b/Makefile index bd50b240..141201ea 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,7 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= BINK_NODE_IMAGE_UPDATE_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update | jq -r '.Digest') \ BINK_NODE_IMAGE_UPDATE2_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update2 | jq -r '.Digest') \ E2E_REGISTRY_USER=$(E2E_REGISTRY_USER) E2E_REGISTRY_PASSWORD=$(E2E_REGISTRY_PASSWORD) \ - go test -timeout 30m -count=1 $(if $(V),-v) $(if $(RUN),-run $(RUN)) . + go test -timeout 40m -count=1 $(if $(V),-v) $(if $(RUN),-run $(RUN)) . ##@ Build diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index fb5ab7b3..845a0730 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -14,7 +14,8 @@ import ( "time" . "github.com/onsi/gomega" - gtypes "github.com/onsi/gomega/types" + "github.com/onsi/gomega/types" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" eventsv1 "k8s.io/api/events/v1" meta "k8s.io/apimachinery/pkg/api/meta" @@ -899,6 +900,170 @@ func TestPullSecretAuth(t *testing.T) { Should(poolAllUpdated(1, digest)) } +// TestControllerRecovery provisions a worker node, starts a rollout, and +// scales the controller deployment to zero while that rollout is actively in +// progress (the node is staging and cannot reboot until the controller drains +// it and flips the desired state). It then scales the controller back up and +// verifies it resumes and completes the interrupted rollout. Covers scenario 5 +// of #69 (kill the controller during a roll-out). +func TestControllerRecovery(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + + env := e2eutil.New(t) + nodeName := env.AddNode(t) + + ctx := context.Background() + + // Phase 1: Create pool with original image and wait for Idle. + pool := env.NewPool("bnp-recovery", env.NodeImageDigestedPullSpec()) + g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) + + g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) { + var bn bootcv1alpha1.BootcNode + err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return bn.Status, err + }).WithTimeout(3 * time.Minute).Should(And( + HaveField("Booted", Not(BeNil())), + HaveField("Conditions", ContainElement(And( + HaveField("Type", bootcv1alpha1.NodeIdle), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.NodeReasonIdle), + ))), + )) + + t.Logf("Node %q is Idle with original image", nodeName) + + // Phase 2: Patch the pool to the update image with the rollout paused. + // Pausing makes the pre-reboot window deterministic: the controller still + // propagates the target to the node spec and the daemon stages it, but a + // paused pool exposes zero reboot slots, so the node parks at Staged and + // cannot reboot on its own. Without pausing, a fast image pull could let + // the controller drain and reboot the node before we scale it down, racing + // the stall assertion below. + updateRef := env.NodeImageUpdateDigestedPullSpec() + + patched := pool.DeepCopy() + patched.Spec.Image.Ref = updateRef + if patched.Spec.Rollout == nil { + patched.Spec.Rollout = &bootcv1alpha1.RolloutSpec{} + } + patched.Spec.Rollout.Paused = true + g.Expect(env.Client.Patch(ctx, patched, client.MergeFrom(pool))).To(Succeed()) + *pool = *patched + + t.Logf("Patched pool to update image %s (paused)", updateRef) + + // Phase 3: Wait until the node has staged the update but has not rebooted. + // The booted image must still be the original digest — the rollout is now + // genuinely in progress but parked pre-reboot. + g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) { + var bn bootcv1alpha1.BootcNode + err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return bn.Status, err + }).WithTimeout(5*time.Minute).Should(And( + HaveField("Staged", And( + Not(BeNil()), + HaveField("ImageDigest", Equal(env.NodeImageUpdateDigest())), + )), + HaveField("Booted", And( + Not(BeNil()), + HaveField("ImageDigest", Equal(env.NodeImageDigest())), + )), + HaveField("Conditions", ContainElement(And( + HaveField("Type", bootcv1alpha1.NodeIdle), + HaveField("Status", metav1.ConditionFalse), + HaveField("Reason", bootcv1alpha1.NodeReasonStaged), + ))), + ), "expected node to stage the update but stay pre-reboot before killing controller") + + t.Logf("Rollout staged and parked pre-reboot; interrupting the controller") + + // Phase 4: Scale the controller to zero, interrupting the rollout. Restore + // it on cleanup in case the test fails while scaled down. + t.Cleanup(func() { scaleController(t, env, ctx, 1) }) + scaleController(t, env, ctx, 0) + t.Logf("Controller scaled to zero mid-rollout") + + // Phase 5: Unpause the pool while the controller is down. The rollout is + // now free to proceed, but nothing can drive it: the node cannot reboot + // into the update until the controller drains it and flips + // DesiredImageState to Booted. The booted image must stay on the original + // digest, proving the interrupted rollout cannot complete on its own and + // that recovery is genuinely driven by the controller coming back. + unpaused := pool.DeepCopy() + unpaused.Spec.Rollout.Paused = false + g.Expect(env.Client.Patch(ctx, unpaused, client.MergeFrom(pool))).To(Succeed()) + *pool = *unpaused + + g.Consistently(func() (string, error) { + var bn bootcv1alpha1.BootcNode + err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + if bn.Status.Booted == nil { + return "", err + } + return bn.Status.Booted.ImageDigest, err + }).WithTimeout(20*time.Second).WithPolling(2*time.Second).Should( + Equal(env.NodeImageDigest()), + "rollout should stall on the original image while the controller is down", + ) + + // Phase 6: Restore the controller. The recovered controller must resume and + // complete the interrupted rollout. + scaleController(t, env, ctx, 1) + t.Logf("Controller restored; waiting for the interrupted rollout to finish") + + g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) { + var bn bootcv1alpha1.BootcNode + err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return bn.Status, err + }).WithTimeout(5*time.Minute).Should(And( + HaveField("Booted", And( + Not(BeNil()), + HaveField("ImageDigest", Equal(env.NodeImageUpdateDigest())), + )), + HaveField("Conditions", ContainElement(And( + HaveField("Type", bootcv1alpha1.NodeIdle), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.NodeReasonIdle), + ))), + ), "expected node to reach Idle with update image after controller recovery") + + t.Logf("Node %q completed the interrupted rollout after controller recovery", nodeName) + + // Verify pool status after recovery. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + Should(poolAllUpdated(1, env.NodeImageUpdateDigest())) +} + +// scaleController scales the operator's controller-manager Deployment to the +// given replica count and waits until its available replicas match. It is used +// to simulate controller outages in recovery tests. +func scaleController(t *testing.T, env *e2eutil.Env, ctx context.Context, replicas int32) { + t.Helper() + + g := NewWithT(t) + deployKey := client.ObjectKey{ + Namespace: testutil.OperatorNamespaceName, + Name: "bootc-operator-controller-manager", + } + + var deploy appsv1.Deployment + g.Expect(env.Client.Get(ctx, deployKey, &deploy)).To(Succeed()) + + scaled := deploy.DeepCopy() + scaled.Spec.Replicas = &replicas + g.Expect(env.Client.Patch(ctx, scaled, client.MergeFrom(&deploy))).To(Succeed()) + + g.Eventually(func() (int32, error) { + var d appsv1.Deployment + err := env.Client.Get(ctx, deployKey, &d) + return d.Status.AvailableReplicas, err + }).WithTimeout(2*time.Minute).Should(Equal(replicas), + "expected controller to scale to %d replica(s)", replicas) +} + func fetchPoolStatus( ctx context.Context, c client.Client, @@ -927,7 +1092,7 @@ func fetchEvents( } } -func poolAllUpdated(nodeCount int32, deployedDigest string) gtypes.GomegaMatcher { +func poolAllUpdated(nodeCount int32, deployedDigest string) types.GomegaMatcher { return And( HaveField("NodeCount", BeEquivalentTo(nodeCount)), HaveField("UpdatedCount", BeEquivalentTo(nodeCount)),