e2e: add daemon-recovery, reboot-timeout, and operator-upgrade tests #174
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -942,3 +942,366 @@ func poolAllUpdated(nodeCount int32, deployedDigest string) gtypes.GomegaMatcher | |
| ))), | ||
| ) | ||
| } | ||
|
|
||
| // expectNodeIdleOnDigest waits until the named node is Idle (no active update | ||
| // cycle) with the given booted image digest. | ||
| func expectNodeIdleOnDigest( | ||
|
Collaborator
There was a problem hiding this comment. I think this function can be reused also in order existing tests. Can you please move the refactoring into a separate commit? |
||
| g Gomega, | ||
| env *e2eutil.Env, | ||
| ctx context.Context, | ||
| nodeName, digest string, | ||
| timeout time.Duration, | ||
| ) { | ||
| g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) { | ||
| var bn bootcv1alpha1.BootcNode | ||
| err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) | ||
| return bn.Status, err | ||
| }).WithTimeout(timeout).Should(And( | ||
| HaveField("Booted", And( | ||
| Not(BeNil()), | ||
| HaveField("ImageDigest", Equal(digest)), | ||
| )), | ||
| HaveField("Conditions", ContainElement(And( | ||
| HaveField("Type", bootcv1alpha1.NodeIdle), | ||
| HaveField("Status", metav1.ConditionTrue), | ||
| HaveField("Reason", bootcv1alpha1.NodeReasonIdle), | ||
| ))), | ||
| ), "expected node %q to be Idle on digest %s", nodeName, digest) | ||
| } | ||
|
|
||
| // daemonPodsOnNode returns a poll function listing the operator daemon pods | ||
| // scheduled on the given node. | ||
| func daemonPodsOnNode( | ||
| env *e2eutil.Env, | ||
| ctx context.Context, | ||
| nodeName string, | ||
| ) func() ([]corev1.Pod, error) { | ||
| return func() ([]corev1.Pod, error) { | ||
| var pods corev1.PodList | ||
| if err := env.Client.List(ctx, &pods, | ||
| client.InNamespace(testutil.OperatorNamespaceName), | ||
| client.MatchingLabels{ | ||
| "app.kubernetes.io/name": "bootc-operator", | ||
| "app.kubernetes.io/component": "daemon", | ||
| }, | ||
| ); err != nil { | ||
| return nil, err | ||
| } | ||
| var onNode []corev1.Pod | ||
| for _, p := range pods.Items { | ||
| if p.Spec.NodeName == nodeName { | ||
| onNode = append(onNode, p) | ||
| } | ||
| } | ||
| return onNode, nil | ||
| } | ||
| } | ||
|
|
||
| // TestDaemonRecovery provisions a worker node, starts a rollout, and deletes | ||
| // the node's daemon pod while the rollout is parked pre-reboot. The DaemonSet | ||
| // must recreate the pod and the rollout must still complete after resume. | ||
| // Covers the daemon half of scenario 5 of #69 (kill the daemon during a | ||
| // roll-out). | ||
| func TestDaemonRecovery(t *testing.T) { | ||
| g := NewWithT(t) | ||
| g.SetDefaultEventuallyTimeout(pollTimeout) | ||
| g.SetDefaultEventuallyPollingInterval(pollInterval) | ||
|
|
||
| env := e2eutil.New(t) | ||
| nodeName := env.AddNode(t) | ||
|
|
||
| ctx := context.Background() | ||
|
|
||
| // Phase 1: Create pool with original image and wait for Idle. | ||
| pool := env.NewPool("bnp-daemonrec", env.NodeImageDigestedPullSpec()) | ||
| g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) | ||
| expectNodeIdleOnDigest(g, env, ctx, nodeName, env.NodeImageDigest(), 3*time.Minute) | ||
| t.Logf("Node %q is Idle with original image", nodeName) | ||
|
|
||
| // Phase 2: Patch to the update image with the rollout paused, so the node | ||
| // deterministically parks at Staged (staging done, pre-reboot) — a stable | ||
| // window in which to kill the daemon. | ||
| updateRef := env.NodeImageUpdateDigestedPullSpec() | ||
|
|
||
| patched := pool.DeepCopy() | ||
| patched.Spec.Image.Ref = updateRef | ||
| if patched.Spec.Rollout == nil { | ||
| patched.Spec.Rollout = &bootcv1alpha1.RolloutSpec{} | ||
| } | ||
| patched.Spec.Rollout.Paused = true | ||
| g.Expect(env.Client.Patch(ctx, patched, client.MergeFrom(pool))).To(Succeed()) | ||
| *pool = *patched | ||
|
|
||
| t.Logf("Patched pool to update image %s (paused)", updateRef) | ||
|
|
||
| // Phase 3: Wait until the node has staged the update but has not rebooted. | ||
| g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) { | ||
| var bn bootcv1alpha1.BootcNode | ||
| err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) | ||
| return bn.Status, err | ||
| }).WithTimeout(5*time.Minute).Should(And( | ||
| HaveField("Staged", And( | ||
| Not(BeNil()), | ||
| HaveField("ImageDigest", Equal(env.NodeImageUpdateDigest())), | ||
| )), | ||
| HaveField("Conditions", ContainElement(And( | ||
| HaveField("Type", bootcv1alpha1.NodeIdle), | ||
| HaveField("Status", metav1.ConditionFalse), | ||
| HaveField("Reason", bootcv1alpha1.NodeReasonStaged), | ||
| ))), | ||
| ), "expected node to stage the update before killing the daemon") | ||
|
Comment on lines
+1026
to
+1052
Collaborator
There was a problem hiding this comment. This can be also refactored in a function and unified with the previous test you implemented in #152 |
||
|
|
||
| // Phase 4: Delete the node's daemon pod mid-rollout. | ||
| g.Eventually(daemonPodsOnNode(env, ctx, nodeName)). | ||
| WithTimeout(2*time.Minute).Should(HaveLen(1), | ||
| "expected one daemon pod on %s", nodeName) | ||
| pods, err := daemonPodsOnNode(env, ctx, nodeName)() | ||
| g.Expect(err).NotTo(HaveOccurred()) | ||
| g.Expect(pods).To(HaveLen(1)) | ||
| oldPod := pods[0] | ||
| oldUID := oldPod.UID | ||
| g.Expect(env.Client.Delete(ctx, &oldPod)).To(Succeed()) | ||
| t.Logf("Deleted daemon pod %q (uid %s) mid-rollout", oldPod.Name, oldUID) | ||
|
|
||
| // Phase 5: The DaemonSet must recreate the daemon pod. | ||
| g.Eventually(daemonPodsOnNode(env, ctx, nodeName)). | ||
| WithTimeout(2*time.Minute).Should(ConsistOf(And( | ||
| HaveField("Status.Phase", corev1.PodRunning), | ||
| Not(HaveField("UID", oldUID)), | ||
| )), "expected a fresh running daemon pod on %s", nodeName) | ||
| t.Logf("Daemon pod recreated; resuming the rollout") | ||
|
|
||
| // Phase 6: Unpause and verify the rollout completes despite the daemon | ||
| // having been restarted mid-rollout. | ||
| unpaused := pool.DeepCopy() | ||
| unpaused.Spec.Rollout.Paused = false | ||
| g.Expect(env.Client.Patch(ctx, unpaused, client.MergeFrom(pool))).To(Succeed()) | ||
| *pool = *unpaused | ||
|
|
||
| expectNodeIdleOnDigest(g, env, ctx, nodeName, env.NodeImageUpdateDigest(), 5*time.Minute) | ||
| g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). | ||
| Should(poolAllUpdated(1, env.NodeImageUpdateDigest())) | ||
| t.Logf("Node %q completed the rollout after daemon recovery", nodeName) | ||
| } | ||
|
|
||
| // TestRebootTimeoutDegraded provisions a worker node, triggers a rollout with a | ||
| // short reboot timeout, and powers the node off once it starts rebooting so it | ||
| // never returns. The controller must report the stuck node as degraded at the | ||
| // pool level once the timeout elapses. Covers scenario 4 of #69 (a faulty node | ||
| // that fails to come back after the reboot). | ||
| func TestRebootTimeoutDegraded(t *testing.T) { | ||
| g := NewWithT(t) | ||
| g.SetDefaultEventuallyTimeout(pollTimeout) | ||
| g.SetDefaultEventuallyPollingInterval(pollInterval) | ||
|
|
||
| env := e2eutil.New(t) | ||
| nodeName := env.AddNode(t) | ||
|
|
||
| ctx := context.Background() | ||
|
|
||
| // Phase 1: Pool with the original image and a short reboot timeout. | ||
| pool := env.NewPool("bnp-reboottmo", env.NodeImageDigestedPullSpec(), | ||
| testutil.WithRebootTimeoutSeconds(90)) | ||
| g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) | ||
| expectNodeIdleOnDigest(g, env, ctx, nodeName, env.NodeImageDigest(), 3*time.Minute) | ||
| t.Logf("Node %q is Idle with original image", nodeName) | ||
|
|
||
| // Phase 2: Patch to the update image to trigger a reboot. | ||
| updateRef := env.NodeImageUpdateDigestedPullSpec() | ||
|
|
||
| patched := pool.DeepCopy() | ||
| patched.Spec.Image.Ref = updateRef | ||
| g.Expect(env.Client.Patch(ctx, patched, client.MergeFrom(pool))).To(Succeed()) | ||
| *pool = *patched | ||
|
|
||
| t.Logf("Patched pool to update image %s", updateRef) | ||
|
|
||
| // Phase 3: Wait for the node to enter Rebooting. The daemon skips | ||
| // reconciliation after issuing the reboot, so this state is durable. | ||
| g.Eventually(func() ([]metav1.Condition, error) { | ||
| var bn bootcv1alpha1.BootcNode | ||
| err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) | ||
| return bn.Status.Conditions, err | ||
| }).WithTimeout(5*time.Minute).Should(ContainElement(And( | ||
| HaveField("Type", bootcv1alpha1.NodeIdle), | ||
| HaveField("Status", metav1.ConditionFalse), | ||
| HaveField("Reason", bootcv1alpha1.NodeReasonRebooting), | ||
| )), "expected node to reach Rebooting state") | ||
|
|
||
| // Phase 4: Power the node off so it never returns from the reboot. This is | ||
| // not racy against the VM actually rebooting: the daemon stops reconciling | ||
| // once it issues the reboot (Phase 3), so the node stays in Rebooting until | ||
| // it either comes back Booted or the reboot timeout trips. Powering it off | ||
| // here guarantees the former never happens, so the timeout path is taken. | ||
| env.PowerOffNode(t, nodeName) | ||
| t.Logf("Node %q powered off while Rebooting; awaiting reboot-timeout", nodeName) | ||
|
|
||
| // Phase 5: Once the reboot timeout elapses the controller must report the | ||
| // stuck node as degraded at the pool level (the downed node cannot | ||
| // self-report while it is off). | ||
| g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). | ||
| WithTimeout(4*time.Minute).Should(And( | ||
| HaveField("DegradedCount", BeEquivalentTo(1)), | ||
| HaveField("Conditions", ContainElement(And( | ||
| HaveField("Type", bootcv1alpha1.PoolDegraded), | ||
| HaveField("Status", metav1.ConditionTrue), | ||
| HaveField("Reason", bootcv1alpha1.PoolNodeDegraded), | ||
| ))), | ||
| ), "expected pool to report the faulty node as degraded") | ||
| t.Logf("Pool reported the faulty node %q as degraded", nodeName) | ||
| } | ||
|
|
||
| // TestDifferentOSUpgrade provisions a worker node on the (Fedora-based) node | ||
| // image, then switches the pool to an image from a different OS lineage (e.g. | ||
| // CentOS Stream) and verifies the node upgrades across the distro boundary. | ||
| // Covers scenario 8 of #69. Skipped when the different-OS image is not seeded | ||
| // (run `make seed-different-os-image`). | ||
| func TestDifferentOSUpgrade(t *testing.T) { | ||
|
Collaborator
There was a problem hiding this comment. Can we remove this test and enable when we have the bink support |
||
| // TODO: A genuinely different-OS-lineage bootc image (e.g. CentOS Stream | ||
| // on a Fedora-based node) does not boot in the current bink VM: the daemon | ||
| // stages the image and bootc switches to it, but the new deployment fails | ||
| // to boot on bink's disk/firmware layout, so bootc auto-rolls-back to the | ||
| // original. The operator then keeps rebooting the node, never observing | ||
| // Booted == target. This is an environmental limitation of the bink rig, | ||
| // not of the operator or this test. Re-enable once a different-OS image | ||
| // that is bootable under bink is available. The Makefile | ||
| // `seed-different-os-image` target and the env plumbing are kept so this | ||
| // test is ready to run again at that point. | ||
| t.Skip("cross-distro upgrade image does not boot under bink (see TODO); skipping") | ||
|
|
||
| g := NewWithT(t) | ||
| g.SetDefaultEventuallyTimeout(pollTimeout) | ||
| g.SetDefaultEventuallyPollingInterval(pollInterval) | ||
|
|
||
| env := e2eutil.New(t) | ||
| if env.NodeImageDifferentOSDigest() == "" { | ||
| t.Skip("BINK_NODE_IMAGE_DIFFERENT_OS_DIGEST not set; run `make seed-different-os-image`") | ||
| } | ||
| nodeName := env.AddNode(t) | ||
|
|
||
| ctx := context.Background() | ||
|
|
||
| // Phase 1: Pool with the original node image; wait for Idle. | ||
| pool := env.NewPool("bnp-diffos", env.NodeImageDigestedPullSpec()) | ||
| g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) | ||
| expectNodeIdleOnDigest(g, env, ctx, nodeName, env.NodeImageDigest(), 3*time.Minute) | ||
| t.Logf("Node %q is Idle with original image", nodeName) | ||
|
|
||
| // Phase 2: Switch the pool to the different-OS image. | ||
| diffRef := env.NodeImageDifferentOSDigestedPullSpec() | ||
|
|
||
| patched := pool.DeepCopy() | ||
| patched.Spec.Image.Ref = diffRef | ||
| g.Expect(env.Client.Patch(ctx, patched, client.MergeFrom(pool))).To(Succeed()) | ||
| *pool = *patched | ||
|
|
||
| t.Logf("Patched pool to different-OS image %s", diffRef) | ||
|
|
||
| // Phase 3: The node must stage, reboot, and come back Idle on the new OS. | ||
| // Allow extra time: this pulls a different, larger base image. | ||
| expectNodeIdleOnDigest(g, env, ctx, nodeName, env.NodeImageDifferentOSDigest(), 8*time.Minute) | ||
| g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). | ||
| Should(poolAllUpdated(1, env.NodeImageDifferentOSDigest())) | ||
| t.Logf("Node %q upgraded to the different-OS image", nodeName) | ||
| } | ||
|
|
||
| // restartOperator simulates an operator upgrade by deleting all operator pods | ||
| // (both the controller Deployment and the DaemonSet). Kubernetes recreates | ||
| // them as it would after an image bump. It waits for a fresh controller pod and | ||
| // a fresh daemon pod on the given node to be Running before returning. | ||
| func restartOperator(t *testing.T, env *e2eutil.Env, ctx context.Context, nodeName string) { | ||
|
Collaborator
There was a problem hiding this comment. we are start having a lot of helper functions, maybe it will be better to move them under test/util/ |
||
| t.Helper() | ||
|
|
||
| g := NewWithT(t) | ||
| g.Expect(env.Client.DeleteAllOf(ctx, &corev1.Pod{}, | ||
| client.InNamespace(testutil.OperatorNamespaceName), | ||
| client.MatchingLabels{"app.kubernetes.io/name": "bootc-operator"}, | ||
| )).To(Succeed()) | ||
|
|
||
| g.Eventually(func() ([]corev1.Pod, error) { | ||
| var pods corev1.PodList | ||
| err := env.Client.List(ctx, &pods, | ||
| client.InNamespace(testutil.OperatorNamespaceName), | ||
| client.MatchingLabels{ | ||
| "app.kubernetes.io/name": "bootc-operator", | ||
| "app.kubernetes.io/component": "controller", | ||
| }, | ||
| ) | ||
| return pods.Items, err | ||
| }).WithTimeout(2*time.Minute).Should(ContainElement( | ||
| HaveField("Status.Phase", corev1.PodRunning), | ||
| ), "expected a running controller pod after restart") | ||
|
|
||
| g.Eventually(daemonPodsOnNode(env, ctx, nodeName)). | ||
| WithTimeout(2*time.Minute).Should(ConsistOf( | ||
| HaveField("Status.Phase", corev1.PodRunning), | ||
| ), "expected a running daemon pod on %s after restart", nodeName) | ||
| } | ||
|
|
||
| // TestOperatorUpgrade brings a node to steady state, simulates an operator | ||
| // upgrade by cycling both operator components, and verifies managed nodes are | ||
| // not disrupted (no extra reboot, still Idle and up to date) and that the | ||
| // operator still drives a subsequent rollout to completion. Covers scenario 9 | ||
| // of #69. | ||
| func TestOperatorUpgrade(t *testing.T) { | ||
| g := NewWithT(t) | ||
| g.SetDefaultEventuallyTimeout(pollTimeout) | ||
| g.SetDefaultEventuallyPollingInterval(pollInterval) | ||
|
|
||
| env := e2eutil.New(t) | ||
| nodeName := env.AddNode(t) | ||
|
|
||
| ctx := context.Background() | ||
|
|
||
| // Phase 1: Reach steady state on the original image. | ||
| pool := env.NewPool("bnp-opupgrade", env.NodeImageDigestedPullSpec()) | ||
| g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) | ||
| expectNodeIdleOnDigest(g, env, ctx, nodeName, env.NodeImageDigest(), 3*time.Minute) | ||
| g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). | ||
| Should(poolAllUpdated(1, env.NodeImageDigest())) | ||
| t.Logf("Node %q is Idle and pool is up to date", nodeName) | ||
|
|
||
| bootsBefore := getBootCount(t, env, ctx, nodeName) | ||
|
|
||
| // Phase 2: Simulate an operator upgrade by cycling both components. | ||
| restartOperator(t, env, ctx, nodeName) | ||
| t.Logf("Operator components restarted") | ||
|
|
||
| // Phase 3: The managed node must not be disrupted by the operator restart: | ||
| // it must stay on its current image without rebooting. | ||
| g.Consistently(func() (string, error) { | ||
| var bn bootcv1alpha1.BootcNode | ||
| err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) | ||
| if bn.Status.Booted == nil { | ||
| return "", err | ||
| } | ||
| return bn.Status.Booted.ImageDigest, err | ||
| }).WithTimeout(30*time.Second).WithPolling(3*time.Second).Should( | ||
| Equal(env.NodeImageDigest()), | ||
| "node should stay on its image across an operator restart", | ||
| ) | ||
|
|
||
| bootsAfter := getBootCount(t, env, ctx, nodeName) | ||
| g.Expect(bootsAfter).To(Equal(bootsBefore), | ||
| "operator restart must not reboot the node") | ||
|
|
||
| expectNodeIdleOnDigest(g, env, ctx, nodeName, env.NodeImageDigest(), 2*time.Minute) | ||
| g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). | ||
| Should(poolAllUpdated(1, env.NodeImageDigest())) | ||
|
|
||
| // Phase 4: The upgraded operator must still function — a subsequent | ||
| // rollout completes normally. | ||
| updateRef := env.NodeImageUpdateDigestedPullSpec() | ||
|
|
||
| patched := pool.DeepCopy() | ||
| patched.Spec.Image.Ref = updateRef | ||
| g.Expect(env.Client.Patch(ctx, patched, client.MergeFrom(pool))).To(Succeed()) | ||
| *pool = *patched | ||
|
|
||
| t.Logf("Patched pool to update image %s after operator upgrade", updateRef) | ||
|
|
||
| expectNodeIdleOnDigest(g, env, ctx, nodeName, env.NodeImageUpdateDigest(), 5*time.Minute) | ||
| g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). | ||
| Should(poolAllUpdated(1, env.NodeImageUpdateDigest())) | ||
| t.Logf("Operator functioned normally after upgrade: rollout completed") | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
How would we make bink working with this image? do we need to change the firmware?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
As mentioned for the test, I would keep this as separate PR once bink is able to work with both