diff --git a/README.md b/README.md index 6e89e288e6..a0b691e924 100644 --- a/README.md +++ b/README.md @@ -274,6 +274,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * [Request Parking](docs/request-parking.md): How the router parks requests through transient worker-pool saturation. * [Sandbox Asset Cache](docs/sandbox-asset-cache.md): How a node caches the sandbox binaries actors run on, decides which runtimes it needs, and evicts to a budget. * [Upgrade Runbook](docs/upgrade.md): Upgrade a running install to a newer supported release, and roll back if needed. +* [SandboxConfig Upgrade](docs/sandboxconfig-upgrade.md): Move pools to a release's SandboxConfigs during an upgrade, and back on rollback. * [Node Storage](docs/node-storage.md): How to give snapshot I/O its own disk. * [Security Guide](docs/security.md): Security overview, security principles, and threat modeling. * [Benchmarking Guide](benchmarking/README.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness. diff --git a/api/infra/k8s/ate/v1/sandboxconfig_types.go b/api/infra/k8s/ate/v1/sandboxconfig_types.go index ab28685d44..e968804d07 100644 --- a/api/infra/k8s/ate/v1/sandboxconfig_types.go +++ b/api/infra/k8s/ate/v1/sandboxconfig_types.go @@ -212,6 +212,25 @@ type SandboxConfigList struct { Items []SandboxConfig `json:"items"` } +// ClassDefaultSandboxConfig returns the default SandboxConfig of class among +// configs, or nil if there is none. It is the newest config of the class with +// SandboxConfigClassDefaultAnnotation set to "true" that is not being deleted. +// Of two created in the same second, the one whose name sorts first wins. +func ClassDefaultSandboxConfig(configs []SandboxConfig, class SandboxClass) *SandboxConfig { + var best *SandboxConfig + for i := range configs { + sc := &configs[i] + if sc.Spec.SandboxClass != class || sc.Annotations[SandboxConfigClassDefaultAnnotation] != "true" || !sc.DeletionTimestamp.IsZero() { + continue + } + if best == nil || best.CreationTimestamp.Before(&sc.CreationTimestamp) || + (sc.CreationTimestamp.Equal(&best.CreationTimestamp) && sc.Name < best.Name) { + best = sc + } + } + return best +} + func init() { SchemeBuilder.Register(&SandboxConfig{}, &SandboxConfigList{}) } diff --git a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go index 6b6663cec9..676559d9e1 100644 --- a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go +++ b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "testing" "time" @@ -136,6 +137,136 @@ func applyVAP(t *testing.T, ctx context.Context) { } } +// removeVAPBinding deletes the shipped policy's binding and waits until the +// apiserver stops enforcing the policy, confirmed by a sentinel that must be +// admitted. +func removeVAPBinding(t *testing.T, ctx context.Context) { + t.Helper() + binding := &unstructured.Unstructured{} + binding.SetAPIVersion("admissionregistration.k8s.io/v1") + binding.SetKind("ValidatingAdmissionPolicyBinding") + binding.SetName("sandboxconfig-assets") + if err := client.IgnoreNotFound(k8sClient.Delete(ctx, binding)); err != nil { + t.Fatalf("delete binding: %v", err) + } + i := 0 + err := wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, 30*time.Second, true, func(ctx context.Context) (bool, error) { + i++ + sc := sandboxConfig(fmt.Sprintf("vap-cooldown-%d", i), SandboxClassGvisor, + map[string]map[string]AssetFile{"amd64": {"other": otherAsset()}}) + if err := k8sClient.Create(ctx, sc); err != nil { + return false, nil + } + return true, k8sClient.Delete(ctx, sc) + }) + if err != nil { + t.Fatalf("VAP did not become inactive: %v", err) + } +} + +// A config that an earlier, looser policy admitted can still be updated, as +// long as the update adds no version and changes no version's assets. +func TestSandboxConfigUpdateUnderStricterPolicy(t *testing.T) { + ctx := t.Context() + runscAssets := func() map[string]map[string]AssetFile { + return map[string]map[string]AssetFile{"amd64": {"runsc": otherAsset()}} + } + // The config before the update. Its versions v1 and v0 have a bare runsc + // asset, which the current policy rejects. + base := func(name string) *SandboxConfig { + sc := withVersions(sandboxConfig(name, SandboxClassGvisor, runscAssets()), sandboxVersion("v0", runscAssets())) + sc.Finalizers = []string{"test.ate.dev/hold"} + sc.Annotations = map[string]string{SandboxConfigClassDefaultAnnotation: "true"} + return sc + } + + tests := []struct { + name string + update func(sc *SandboxConfig) + errMsg string // empty when the update is allowed + }{{ + name: "remove a finalizer", + update: func(sc *SandboxConfig) { sc.Finalizers = nil }, + }, { + name: "remove the class-default annotation", + update: func(sc *SandboxConfig) { sc.Annotations = nil }, + }, { + name: "disable a version", + update: func(sc *SandboxConfig) { sc.Spec.Versions[1].State = ptr.To(SandboxVersionStateDisabled) }, + }, { + name: "delete a version", + update: func(sc *SandboxConfig) { sc.Spec.Versions = sc.Spec.Versions[:1] }, + }, { + name: "change defaultVersion", + update: func(sc *SandboxConfig) { withDefaultVersion(sc, "v0") }, + }, { + name: "add a valid version", + update: func(sc *SandboxConfig) { withVersions(sc, sandboxVersion("v2", gvisorAssets())) }, + }, { + name: "add an invalid version", + update: func(sc *SandboxConfig) { withVersions(sc, sandboxVersion("v2", runscAssets())) }, + errMsg: gvisorAssetMsg, + }, { + name: "change a version's assets", + update: func(sc *SandboxConfig) { + sc.Spec.Versions[1].Assets["arm64"] = map[string]AssetFile{"runsc": otherAsset()} + }, + errMsg: gvisorAssetMsg, + }, { + name: "change sandboxClass", + update: func(sc *SandboxConfig) { sc.Spec.SandboxClass = SandboxClassMicroVM }, + errMsg: "microvm SandboxConfig must define", + }} + + // Create the configs while no policy is enforced, as an earlier release + // might have. + removeVAPBinding(t, ctx) + configs := make([]*SandboxConfig, len(tests)) + for i := range tests { + configs[i] = base(fmt.Sprintf("stricter-policy-%d", i)) + if err := k8sClient.Create(ctx, configs[i]); err != nil { + t.Fatalf("Create() %s: %v", configs[i].Name, err) + } + } + applyVAP(t, ctx) + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sc := configs[i] + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + latest := &SandboxConfig{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(sc), latest); err != nil { + t.Errorf("cleanup: get SandboxConfig %s: %v", sc.Name, err) + return + } + latest.Finalizers = nil + if err := k8sClient.Update(ctx, latest); err != nil { + t.Errorf("cleanup: remove finalizer from SandboxConfig %s: %v", sc.Name, err) + } + if err := client.IgnoreNotFound(k8sClient.Delete(ctx, latest)); err != nil { + t.Errorf("cleanup: delete SandboxConfig %s: %v", sc.Name, err) + } + }) + tt.update(sc) + err := k8sClient.Update(ctx, sc) + if tt.errMsg == "" { + if err != nil { + t.Fatalf("Update() unexpected error: %v", err) + } + return + } + if err == nil { + t.Fatalf("Update() succeeded, want denied") + } + if !strings.Contains(err.Error(), tt.errMsg) { + t.Errorf("Update() error = %q, want it to contain %q", err.Error(), tt.errMsg) + } + }) + } +} + func TestSandboxConfigValidation(t *testing.T) { ctx := t.Context() applyVAP(t, ctx) @@ -344,14 +475,15 @@ func TestSandboxConfigUpdateRemovingDefaultVersion(t *testing.T) { } } -// TestShippedSandboxConfigManifests guards the SandboxConfigs the install ships -// against the CRD schema and the shipped ValidatingAdmissionPolicy. +// TestShippedSandboxConfigManifests guards the SandboxConfig templates the +// install renders against the CRD schema and the shipped +// ValidatingAdmissionPolicy. func TestShippedSandboxConfigManifests(t *testing.T) { ctx := t.Context() applyVAP(t, ctx) for _, path := range []string{ - "../../../../../manifests/ate-install/sandboxconfig-gvisor.yaml", + "../../../../../manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl", "../../../../../manifests/microvm/sandboxconfig-microvm.yaml.tmpl", } { t.Run(path, func(t *testing.T) { @@ -359,13 +491,14 @@ func TestShippedSandboxConfigManifests(t *testing.T) { if err != nil { t.Fatalf("read manifest: %v", err) } - raw = []byte(strings.NewReplacer("${BUCKET_SCHEME}", "s3", "${BUCKET_NAME}", "test-bucket", "${ASSET_BUCKET_NAME}", "test-bucket").Replace(string(raw))) + raw = []byte(strings.NewReplacer("${SANDBOXCONFIG_NAME}", "test-"+filepath.Base(path), "${SUBSTRATE_VERSION}", "v0.4.2", "${BUCKET_SCHEME}", "s3", "${ASSET_BUCKET_NAME}", "test-bucket").Replace(string(raw))) sc := &SandboxConfig{} if err := yaml.UnmarshalStrict(raw, sc); err != nil { t.Fatalf("decode manifest: %v", err) } - if sc.Annotations[SandboxConfigClassDefaultAnnotation] != "true" { - t.Errorf("manifest is not annotated %s=true", SandboxConfigClassDefaultAnnotation) + // The installers add the annotation when they create the config. + if _, ok := sc.Annotations[SandboxConfigClassDefaultAnnotation]; ok { + t.Errorf("template sets %s; the installers add it", SandboxConfigClassDefaultAnnotation) } if err := k8sClient.Create(ctx, sc); err != nil { t.Fatalf("Create() unexpected error: %v", err) diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 30a9dfe83c..c0ee3edc7b 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -32,7 +32,8 @@ To run the cold resume tests from your workstation instead, see Envoy ingress router. For a `microvm` test the orchestrator also runs `hack/install-microvm-deps.sh --install` between the two, which stages kata + cloud-hypervisor + virtiofsd assets to the cluster's object - store bucket and applies the cluster-wide `microvm` SandboxConfig. + store bucket and creates this release's microVM SandboxConfig as the + class default. For a `nighthawk-ingress` test the orchestrator additionally patches the `atenet-router` Deployment right after `deploy_substrate`: `envoyCpu` is the benchmark's independent variable and the shipped manifest sets @@ -49,7 +50,8 @@ To run the cold resume tests from your workstation instead, see variable (a JSON object) if set. A `--metadata` in the test's own `flags` is merged in too, its keys winning on conflict. - Polls until complete/failed/timeout; tails logs; deletes the Job. - - Tears down workloads + microVM deps (if any) + substrate. + - Tears down workloads + substrate. Deleting substrate also removes the + microVM SandboxConfigs with their CRD. - If not the last test, redeploys them so the next run starts clean. ## Choosing a sandbox class diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index c50fdc2946..6f5234346f 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -421,19 +421,13 @@ def teardown_substrate() -> None: def install_microvm_deps() -> None: - """Stage kata/cloud-hypervisor assets and apply the cluster-wide - microvm SandboxConfig. Required before a microvm WorkerPool can run - actors; must run after deploy_substrate() (which installs the CRDs).""" + """Stage kata/cloud-hypervisor assets and create this release's microVM + SandboxConfig as the class default. Required before a microVM WorkerPool + can run actors; must run after deploy_substrate() (which installs the + CRDs).""" run(["hack/install-microvm-deps.sh", "--install"]) -def teardown_microvm_deps() -> None: - """Remove the microvm SandboxConfig. Must run before - teardown_substrate(), which deletes the SandboxConfig CRD (and would - prevent this from succeeding via kubectl).""" - run_no_check(["hack/install-microvm-deps.sh", "--delete"]) - - def deploy_workloads( worker_count: int = 1, sandbox_class: str = "gvisor", @@ -632,11 +626,9 @@ def main() -> None: # fire that crashed mid-test (or any other process that left # state behind) would otherwise leak its substrate + workloads # into this run. All teardowns use --ignore-not-found, so - # this is cheap on a clean cluster. Order matters: - # microvm-deps deletes a SandboxConfig CR, which requires the - # SandboxConfig CRD that teardown_substrate removes. + # this is cheap on a clean cluster. teardown_substrate also + # removes the microVM SandboxConfigs with their CRD. teardown_workloads() - teardown_microvm_deps() teardown_substrate() sandbox_class = test.get("sandboxClass", "gvisor") @@ -678,10 +670,7 @@ def main() -> None: finally: # Always tear down, even if deploy or run failed, so the # next test (and the next CronJob fire) starts clean. - # microvm-deps must go before substrate for the same reason - # as above. teardown_workloads() - teardown_microvm_deps() teardown_substrate() duration = time.time() - start_time results.append((test["name"], status, duration, failure_msg)) diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 87672d5217..bb7446439b 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -113,6 +113,12 @@ apiserver, the controller, atenet, and atelet. It creates every `create` resource below on the way, so those subcommands are only needed to redo one on a running cluster. +`deploy ate-system` and `deploy sandboxconfig` create this build's gVisor +SandboxConfig with the `sandboxconfig.ate.dev/is-class-default` annotation. It +is named `substrate--gvisor`, hashed from the rendered config. Pass +`--create-with-default-annotation=false` to create it without the annotation. +An existing config is left as it is, annotations included. + ## Publish - `publish worker-images` diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 0d9f38c8eb..d312ab5061 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -84,6 +84,9 @@ Shape the install with the global --atenet-dataplane, --cluster-size, if err != nil { return err } + if opts.CreateSandboxConfigWithDefaultAnnotation, err = cmd.Flags().GetBool(createWithDefaultAnnotationFlag); err != nil { + return err + } return env.DeployAteSystem(cmd.Context(), opts) }, } @@ -136,12 +139,36 @@ var deployPodCertControllerCmd = &cobra.Command{ }, } +// createWithDefaultAnnotationFlag names the flag that leaves the +// class-default annotation off this build's gVisor SandboxConfig when false. +// It is a plain flag rather than a recorded setting, because it applies only +// to the run that creates the config. +const createWithDefaultAnnotationFlag = "create-with-default-annotation" + +const createWithDefaultAnnotationUsage = "Add the sandboxconfig.ate.dev/is-class-default annotation to this build's gVisor SandboxConfig if this run creates it. An existing config is left as it is" + var deploySandboxConfigCmd = &cobra.Command{ Use: "sandboxconfig", - Short: "Deploy the SandboxConfig admission policy and the default gVisor SandboxConfig only", - Args: cobra.NoArgs, + Short: "Deploy the SandboxConfig admission policy and this build's gVisor SandboxConfig only", + Long: `Deploy the SandboxConfig admission policy and create this build's gVisor +SandboxConfig from manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl. It is +named substrate--gvisor, hashed from the rendered config, so builds with +the same config share one, and labeled +sandboxconfig.ate.dev/created-by-version with the version that creates it. + +When this run creates the config, it gets the +sandboxconfig.ate.dev/is-class-default annotation, unless +--create-with-default-annotation=false. An existing config is left as it is, +annotations included. Change its annotation with kubectl annotate. Every +gVisor WorkerPool without a configRef follows the newest config with the +annotation, so a warning names a newer one if it exists.`, + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - return env.DeploySandboxConfig(cmd.Context()) + create, err := cmd.Flags().GetBool(createWithDefaultAnnotationFlag) + if err != nil { + return err + } + return env.DeploySandboxConfig(cmd.Context(), create) }, } @@ -181,4 +208,6 @@ func init() { }) config.BindCommandFlags(deployAteSystemPath, deployAteSystemCmd.Flags()) deployAteSystemCmd.Flags().Lookup("setup-csi").NoOptDefVal = "none" + deployAteSystemCmd.Flags().Bool(createWithDefaultAnnotationFlag, true, createWithDefaultAnnotationUsage) + deploySandboxConfigCmd.Flags().Bool(createWithDefaultAnnotationFlag, true, createWithDefaultAnnotationUsage) } diff --git a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go index 52a5036e62..16fe9fa899 100644 --- a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go +++ b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go @@ -14,7 +14,7 @@ // Package countermicrovm installs the micro-VM variant of the counter demo. // -// It needs the cluster-wide `microvm` SandboxConfig that +// It needs the microVM class default SandboxConfig that // hack/install-microvm-deps.sh --install creates. It has no external-volume // option: the CSI path is covered by demo-counter. package countermicrovm diff --git a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go index 7c8146f494..4893e1a346 100644 --- a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go +++ b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go @@ -14,7 +14,7 @@ // Package egressmicrovm installs the micro-VM variant of the egress demo. // -// It needs the cluster-wide `microvm` SandboxConfig that +// It needs the microVM class default SandboxConfig that // hack/install-microvm-deps.sh --install creates. It is a demo of its own // rather than a flag on demo-egress so that it appears in help and in the // `delete all` sweep, and so that both can be installed side by side: the diff --git a/cmd/ate-setup/internal/images/release.go b/cmd/ate-setup/internal/images/release.go index 75d30f1213..0510989d09 100644 --- a/cmd/ate-setup/internal/images/release.go +++ b/cmd/ate-setup/internal/images/release.go @@ -48,7 +48,8 @@ const RevisionLabel = "org.opencontainers.image.revision" const VersionLabel = "org.opencontainers.image.version" // releaseTag matches a release tag, vMAJOR.MINOR.PATCH, or a release candidate, -// vMAJOR.MINOR.PATCH-PRERELEASE. hack/release/draft-release.sh accepts the same. +// vMAJOR.MINOR.PATCH-PRERELEASE. hack/release/draft-release.sh accepts the +// same. var releaseTag = regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$`) // CheckReleaseVersion returns an error unless version is a release tag. diff --git a/cmd/ate-setup/internal/kube/apply.go b/cmd/ate-setup/internal/kube/apply.go index 9d92dbb08f..e1ee74b65f 100644 --- a/cmd/ate-setup/internal/kube/apply.go +++ b/cmd/ate-setup/internal/kube/apply.go @@ -65,11 +65,11 @@ func (c *Client) Apply(ctx context.Context, objs []*unstructured.Unstructured) e return nil } -// applyBackoff paces the retries of a failed apply. Five attempts over roughly -// eight seconds covers the two things that routinely fail here — a validating -// webhook whose pod is still starting, and a namespace or CRD whose creation -// has not reached the apiserver handling this request — without turning a -// genuinely broken cluster into a long wait. +// applyBackoff paces the retries of a failed apply or create. Five attempts +// over roughly eight seconds covers the two things that routinely fail here — +// a validating webhook whose pod is still starting, and a namespace or CRD +// whose creation has not reached the apiserver handling this request — without +// turning a genuinely broken cluster into a long wait. var applyBackoff = wait.Backoff{ Steps: 5, Duration: 500 * time.Millisecond, @@ -111,6 +111,34 @@ func applyWithRetry(ctx context.Context, ri dynamicResource, obj *unstructured.U return nil } +// CreateOne creates obj unless an object of its name exists, retrying the +// failures that clear on their own. It reports whether it created obj. +func (c *Client) CreateOne(ctx context.Context, obj *unstructured.Unstructured) (bool, error) { + ri, err := c.resourceFor(obj) + if err != nil { + return false, err + } + return createWithRetry(ctx, ri, obj) +} + +func createWithRetry(ctx context.Context, ri dynamicResource, obj *unstructured.Unstructured) (bool, error) { + retryable := func(err error) bool { + return ctx.Err() == nil && retryableWaitError(err) + } + err := retry.OnError(applyBackoff, retryable, func() error { + _, err := ri.Create(ctx, obj, metav1.CreateOptions{FieldManager: FieldManager}) + return err + }) + // A retry after a lost response can find the object the first attempt + // created, and so reports false. + if apierrors.IsAlreadyExists(err) { + return false, nil + } else if err != nil { + return false, fmt.Errorf("while creating %s: %w", Describe(obj), err) + } + return true, nil +} + // retryableApplyError reports whether a failed apply is worth re-sending. func retryableApplyError(err error) bool { // Apply is forced, so it never loses a field-ownership argument. A conflict @@ -271,6 +299,7 @@ func (c *Client) resourceFor(obj *unstructured.Unstructured) (dynamicResource, e // namespaced and cluster-scoped clients both satisfy it. type dynamicResource interface { Apply(ctx context.Context, name string, obj *unstructured.Unstructured, opts metav1.ApplyOptions, subresources ...string) (*unstructured.Unstructured, error) + Create(ctx context.Context, obj *unstructured.Unstructured, opts metav1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) Delete(ctx context.Context, name string, opts metav1.DeleteOptions, subresources ...string) error Get(ctx context.Context, name string, opts metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) } diff --git a/cmd/ate-setup/internal/kube/apply_test.go b/cmd/ate-setup/internal/kube/apply_test.go index ee73adf75f..58632de630 100644 --- a/cmd/ate-setup/internal/kube/apply_test.go +++ b/cmd/ate-setup/internal/kube/apply_test.go @@ -62,7 +62,8 @@ func TestDeletePathReportsUnparseableManifest(t *testing.T) { } } -// stubResource is a dynamicResource whose Apply returns queued results. +// stubResource is a dynamicResource whose Apply and Create return queued +// results. type stubResource struct { results []error attempts int @@ -76,6 +77,14 @@ func (s *stubResource) Apply(context.Context, string, *unstructured.Unstructured return nil, nil } +func (s *stubResource) Create(context.Context, *unstructured.Unstructured, metav1.CreateOptions, ...string) (*unstructured.Unstructured, error) { + s.attempts++ + if s.attempts <= len(s.results) { + return nil, s.results[s.attempts-1] + } + return nil, nil +} + func (s *stubResource) Delete(context.Context, string, metav1.DeleteOptions, ...string) error { return nil } @@ -158,3 +167,51 @@ func TestApplyWithRetryStopsOnCancellation(t *testing.T) { t.Errorf("apply attempted %d times, want 1", ri.attempts) } } + +func TestCreateWithRetry(t *testing.T) { + shortApplyBackoff(t) + alreadyExists := apierrors.NewAlreadyExists(schema.GroupResource{Resource: "configmaps"}, "ate-api-server-envvars") + unavailable := apierrors.NewServiceUnavailable("apiserver is starting") + + tests := []struct { + name string + results []error + cancelled bool + wantCreated bool + wantErr bool + wantAttempts int + }{ + {name: "created", wantCreated: true, wantAttempts: 1}, + {name: "exists", results: []error{alreadyExists}, wantAttempts: 1}, + {name: "created after a transient failure", results: []error{unavailable}, wantCreated: true, wantAttempts: 2}, + {name: "exists after a transient failure", results: []error{unavailable, alreadyExists}, wantAttempts: 2}, + { + name: "rejected", + results: []error{apierrors.NewInvalid(schema.GroupKind{Kind: "ConfigMap"}, "ate-api-server-envvars", nil)}, + wantErr: true, + wantAttempts: 1, + }, + // Retrying past a cancelled context turns ^C into an eight-second wait. + {name: "cancelled", results: []error{unavailable, unavailable}, cancelled: true, wantErr: true, wantAttempts: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if tt.cancelled { + cancel() + } + ri := &stubResource{results: tt.results} + created, err := createWithRetry(ctx, ri, testConfigMap()) + if (err != nil) != tt.wantErr { + t.Fatalf("createWithRetry() error = %v, wantErr %v", err, tt.wantErr) + } + if created != tt.wantCreated { + t.Errorf("createWithRetry() = %v, want %v", created, tt.wantCreated) + } + if ri.attempts != tt.wantAttempts { + t.Errorf("create attempted %d times, want %d", ri.attempts, tt.wantAttempts) + } + }) + } +} diff --git a/cmd/ate-setup/internal/steps/benchmarks.go b/cmd/ate-setup/internal/steps/benchmarks.go index 68c51d020c..b21ac88f2c 100644 --- a/cmd/ate-setup/internal/steps/benchmarks.go +++ b/cmd/ate-setup/internal/steps/benchmarks.go @@ -70,9 +70,9 @@ func (e *Env) DeployBenchmarks(ctx context.Context, opts BenchmarkOptions) error } log.Stepf("deploy_benchmarks (worker_count=%d, sandbox_class=%s)", opts.WorkerCount, opts.SandboxClass) - // The microvm SandboxConfig lives outside the default set installed by - // `deploy ate-system`, which only installs gvisor-default. The workloads - // deploy references it by name and would fail if this were skipped. + // `deploy ate-system` installs only the gVisor SandboxConfig. The + // microVM workload pool follows the microVM class default and would not + // resolve if this were skipped. if opts.SandboxClass == config.SandboxClassMicrovm { if err := e.runScript(ctx, installMicrovmDepScript, "--install"); err != nil { return err @@ -110,17 +110,9 @@ func (e *Env) DeleteBenchmarks(ctx context.Context, opts BenchmarkOptions) error } log.Stepf("delete_benchmarks (sandbox_class=%s)", opts.SandboxClass) - if err := e.runScript(ctx, deployLocustScript, "--delete"); err != nil { - return err - } - // The microvm SandboxConfig is cluster-wide. It is only torn down when - // the caller opted into microvm, and only after the benchmark pool is - // gone. The script leaves it in place if another pool still uses it. - // `delete all` tries again once every demo is gone. - if opts.SandboxClass == config.SandboxClassMicrovm { - return e.runScript(ctx, installMicrovmDepScript, "--delete") - } - return nil + // The microVM SandboxConfigs stay. They are cluster-wide and go with the + // SandboxConfig CRD when the control plane is deleted. + return e.runScript(ctx, deployLocustScript, "--delete") } // runScript executes a repository script from the repository root with the @@ -128,7 +120,9 @@ func (e *Env) DeleteBenchmarks(ctx context.Context, opts BenchmarkOptions) error func (e *Env) runScript(ctx context.Context, relPath string, args ...string) error { cmd := exec.CommandContext(ctx, e.Cfg.Path(relPath), args...) cmd.Dir = e.Cfg.Root - cmd.Env = e.Cfg.ScriptEnv() + // install-microvm-deps.sh labels its SandboxConfig with VERSION. Set it + // so the label matches the gVisor config's, also with prebuilt images. + cmd.Env = append(e.Cfg.ScriptEnv(), "VERSION="+e.SubstrateVersion()) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin diff --git a/cmd/ate-setup/internal/steps/delete.go b/cmd/ate-setup/internal/steps/delete.go index 7223e8a332..5c43fb528e 100644 --- a/cmd/ate-setup/internal/steps/delete.go +++ b/cmd/ate-setup/internal/steps/delete.go @@ -207,38 +207,14 @@ type Deleter interface { Delete(ctx context.Context, e *Env) error } -// DeleteAll removes every demo, then the microvm SandboxConfig, then the -// control plane. +// DeleteAll removes every demo, then the control plane. func (e *Env) DeleteAll(ctx context.Context, demos []Deleter) error { log.Step("delete_all") - if err := e.deleteDemos(ctx, demos); err != nil { - return err - } - return e.DeleteAteSystem(ctx) -} - -// deleteDemos removes every demo, then tries to delete the microvm -// SandboxConfig. An earlier `delete benchmarks` or `delete demo` may have -// left the config behind because a pool still used it. After the demos, no -// pool does. -// -// The delete is best effort. The script skips a config that is still in use, -// and a failure is only logged, because DeleteAteSystem deletes the config -// with the CRD anyway. -// -// The gvisor config needs no such step. It is installed and deleted with the -// control plane, and DeleteAteSystem strips its finalizer itself. The microvm -// config is managed by hack/install-microvm-deps.sh, whose `kubectl delete` -// would block while a pool still uses it. -func (e *Env) deleteDemos(ctx context.Context, demos []Deleter) error { for _, demo := range demos { if err := demo.Delete(ctx, e); err != nil { return err } } - if err := e.runScript(ctx, installMicrovmDepScript, "--delete"); err != nil { - log.Warnf("while deleting the microvm SandboxConfig: %v", err) - } - return nil + return e.DeleteAteSystem(ctx) } diff --git a/cmd/ate-setup/internal/steps/delete_test.go b/cmd/ate-setup/internal/steps/delete_test.go index 9274cadc6f..a255a23b70 100644 --- a/cmd/ate-setup/internal/steps/delete_test.go +++ b/cmd/ate-setup/internal/steps/delete_test.go @@ -18,8 +18,6 @@ import ( "context" "errors" "fmt" - "os" - "path/filepath" "slices" "strings" "testing" @@ -37,7 +35,6 @@ import ( "k8s.io/client-go/util/retry" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" - "go.ate.dev/substrate/cmd/ate-setup/internal/config" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/internal/clustertrustbundle" ) @@ -193,92 +190,6 @@ func TestReleaseSandboxConfigs(t *testing.T) { } } -// recordingDeleter is a Deleter that appends its name to a log file shared -// with the fake microvm script, so a test can read back the order of calls. -type recordingDeleter struct { - name string - log string - err error -} - -func (d recordingDeleter) Delete(context.Context, *Env) error { - appendLine(d.log, d.name) - return d.err -} - -func appendLine(path, line string) { - f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil { - panic(err) - } - defer f.Close() - fmt.Fprintln(f, line) -} - -func TestDeleteDemos(t *testing.T) { - demoErr := errors.New("demo failed") - tests := []struct { - name string - // demos are the deleters in order. A nil entry's err is nil. - demos []recordingDeleter - // scriptExit is the exit code of the fake install-microvm-deps.sh. - scriptExit int - wantCalls []string - wantErr error - }{ - { - name: "microvm SandboxConfig is deleted after every demo", - demos: []recordingDeleter{{name: "demo-a"}, {name: "demo-b"}}, - wantCalls: []string{"demo-a", "demo-b", "script --delete"}, - }, - { - name: "script failure is logged, not returned", - demos: []recordingDeleter{{name: "demo-a"}}, - scriptExit: 1, - wantCalls: []string{"demo-a", "script --delete"}, - }, - { - name: "a failing demo stops before the script", - demos: []recordingDeleter{{name: "demo-a", err: demoErr}, {name: "demo-b"}}, - wantCalls: []string{"demo-a"}, - wantErr: demoErr, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - root := t.TempDir() - calls := filepath.Join(root, "calls.log") - script := filepath.Join(root, installMicrovmDepScript) - if err := os.MkdirAll(filepath.Dir(script), 0o755); err != nil { - t.Fatal(err) - } - body := fmt.Sprintf("#!/bin/sh\necho \"script $*\" >> %q\nexit %d\n", calls, tt.scriptExit) - if err := os.WriteFile(script, []byte(body), 0o755); err != nil { - t.Fatal(err) - } - demos := make([]Deleter, 0, len(tt.demos)) - for _, d := range tt.demos { - d.log = calls - demos = append(demos, d) - } - e := &Env{Cfg: &config.Config{Root: root}} - - err := e.deleteDemos(t.Context(), demos) - if !errors.Is(err, tt.wantErr) { - t.Fatalf("deleteDemos() error = %v, want %v", err, tt.wantErr) - } - got, _ := os.ReadFile(calls) - gotCalls := strings.Fields(strings.ReplaceAll(strings.TrimSpace(string(got)), "script --delete", "script_--delete")) - for i := range gotCalls { - gotCalls[i] = strings.ReplaceAll(gotCalls[i], "script_--delete", "script --delete") - } - if !slices.Equal(gotCalls, tt.wantCalls) { - t.Errorf("calls = %q, want %q", gotCalls, tt.wantCalls) - } - }) - } -} - // trustBundle returns a bundle for signer the way the API server requires it // to be named: the signer name with "/" replaced by ":", then a suffix. func trustBundle(signer, suffix string) *certsv1.ClusterTrustBundle { diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index bacf30a4e5..03f07997d0 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -43,6 +43,9 @@ type DeployOptions struct { // SetupCSI additionally installs the CSI driver (nfs, hostpath, both, none). // Kind only. The hostpath driver is Kind only. SetupCSI string + // CreateSandboxConfigWithDefaultAnnotation adds the class-default + // annotation to this build's gVisor SandboxConfig if the deploy creates it. + CreateSandboxConfigWithDefaultAnnotation bool } // Validate checks the options that can be checked without configuration or a @@ -86,6 +89,11 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err := e.requireNoWorkerPods(ctx); err != nil { return err } + // Checked here so a refusal does not leave a half-upgraded system. + sc, err := e.checkSandboxConfig(ctx) + if err != nil { + return err + } // The namespace has to exist before RBAC or CRDs are applied. if err := e.EnsureAteSystemNamespace(ctx); err != nil { @@ -112,7 +120,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } - if err := e.DeploySandboxConfig(ctx); err != nil { + if err := e.deploySandboxConfig(ctx, sc, opts.CreateSandboxConfigWithDefaultAnnotation); err != nil { return err } @@ -581,26 +589,6 @@ func (e *Env) DeployAtenet(ctx context.Context) error { return nil } -// DeploySandboxConfig applies the SandboxConfig admission policy, then the -// default gVisor SandboxConfig. -func (e *Env) DeploySandboxConfig(ctx context.Context) error { - log.Step("deploy_sandboxconfig") - - if err := e.EnsureCRDs(ctx); err != nil { - return err - } - - // Enforce per-class SandboxConfig asset requirements. This is applied - // before any SandboxConfig so the config below is validated too. - if err := e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-validation.yaml")); err != nil { - return err - } - - // Install the cluster-wide default gVisor SandboxConfig that gVisor - // WorkerPools resolve by default unless they pin their own configRef. - return e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-gvisor.yaml")) -} - // EnsureCRDs creates the generated CRDs that are missing. Existing CRDs and // RBAC change only through DeployCRDs, so a component redeploy cannot move them. func (e *Env) EnsureCRDs(ctx context.Context) error { diff --git a/cmd/ate-setup/internal/steps/sandboxconfig.go b/cmd/ate-setup/internal/steps/sandboxconfig.go new file mode 100644 index 0000000000..72c36d2e73 --- /dev/null +++ b/cmd/ate-setup/internal/steps/sandboxconfig.go @@ -0,0 +1,201 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "slices" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/dynamic" + + atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/render" +) + +// DeploySandboxConfig applies the SandboxConfig admission policy, then +// creates this build's gVisor SandboxConfig from sandboxconfig-gvisor.yaml.tmpl +// unless it exists. sandboxConfigName says how the config is named. A config +// it creates gets the class-default annotation when +// createWithDefaultAnnotation is true. An existing config, annotations +// included, is left as it is. +func (e *Env) DeploySandboxConfig(ctx context.Context, createWithDefaultAnnotation bool) error { + sc, err := e.checkSandboxConfig(ctx) + if err != nil { + return err + } + return e.deploySandboxConfig(ctx, sc, createWithDefaultAnnotation) +} + +// deploySandboxConfig is DeploySandboxConfig for the config that +// checkSandboxConfig returned. +func (e *Env) deploySandboxConfig(ctx context.Context, sc *unstructured.Unstructured, createWithDefaultAnnotation bool) error { + log.Step("deploy_sandboxconfig") + + if err := e.EnsureCRDs(ctx); err != nil { + return err + } + + // Enforce per-class SandboxConfig asset requirements. This is applied + // before any SandboxConfig so the config below is validated too. + if err := e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-validation.yaml")); err != nil { + return err + } + + if createWithDefaultAnnotation { + annotations := sc.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[atev1.SandboxConfigClassDefaultAnnotation] = "true" + sc.SetAnnotations(annotations) + } + // Create, not apply: an existing config may have been edited since, so it + // is left as it is. + created, err := e.Kube.CreateOne(ctx, sc) + if err != nil { + return err + } + if !created { + log.Infof("SandboxConfig %s exists; leaving it and its annotations as they are", sc.GetName()) + } else if createWithDefaultAnnotation { + log.Infof("created SandboxConfig %s with %s=true", sc.GetName(), atev1.SandboxConfigClassDefaultAnnotation) + } else { + log.Infof("created SandboxConfig %s", sc.GetName()) + } + // Checked after the create, against the config's real creation time and + // annotations. + configs, err := e.listSandboxConfigs(ctx) + if err != nil { + return err + } + if warning := classDefaultWarning(configs, sc.GetName(), atev1.SandboxClassGvisor); warning != "" { + log.Warnf("%s", warning) + } + return nil +} + +// checkSandboxConfig returns this build's gVisor SandboxConfig, labeled with +// the Substrate version that creates it. It fails when the version is not a +// valid label value, or when requireSandboxConfigNotDeleting fails. +func (e *Env) checkSandboxConfig(ctx context.Context) (*unstructured.Unstructured, error) { + version := e.SubstrateVersion() + if errs := validation.IsValidLabelValue(version); len(errs) > 0 { + return nil, fmt.Errorf("version %q is not a valid SandboxConfig label value: %s; set VERSION to one", version, strings.Join(errs, "; ")) + } + tmpl := e.Cfg.Manifest("sandboxconfig-gvisor.yaml.tmpl") + unnamed, err := render.Template(tmpl, nil, nil) + if err != nil { + return nil, err + } + name := sandboxConfigName(atev1.SandboxClassGvisor, unnamed) + configs, err := e.listSandboxConfigs(ctx) + if err != nil { + return nil, err + } + if err := requireSandboxConfigNotDeleting(configs, name); err != nil { + return nil, err + } + manifest := render.Expand(string(unnamed), map[string]string{"SANDBOXCONFIG_NAME": name, "SUBSTRATE_VERSION": version}, nil) + objs, err := kube.DecodeManifestBytes(manifest) + if err != nil { + return nil, err + } + if len(objs) != 1 { + return nil, fmt.Errorf("%s holds %d objects, want 1 SandboxConfig", tmpl, len(objs)) + } + return objs[0], nil +} + +// requireSandboxConfigNotDeleting fails when configs hold the SandboxConfig +// named name and it is being deleted, because the create would find the +// terminating object and leave it. +func requireSandboxConfigNotDeleting(configs []atev1.SandboxConfig, name string) error { + i := slices.IndexFunc(configs, func(sc atev1.SandboxConfig) bool { return sc.Name == name }) + if i >= 0 && !configs[i].DeletionTimestamp.IsZero() { + return fmt.Errorf("SandboxConfig %s is being deleted; wait until it is gone, then run this again", name) + } + return nil +} + +// listSandboxConfigs returns the SandboxConfigs in the cluster. A cluster +// without the SandboxConfig CRD has none. +func (e *Env) listSandboxConfigs(ctx context.Context) ([]atev1.SandboxConfig, error) { + list, err := e.sandboxConfigs().List(ctx, metav1.ListOptions{}) + if apierrors.IsNotFound(err) { + return nil, nil + } else if err != nil { + return nil, fmt.Errorf("while listing SandboxConfigs: %w", err) + } + configs := make([]atev1.SandboxConfig, len(list.Items)) + for i := range list.Items { + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(list.Items[i].Object, &configs[i]); err != nil { + return nil, fmt.Errorf("while decoding SandboxConfig %s: %w", list.Items[i].GetName(), err) + } + } + return configs, nil +} + +func (e *Env) sandboxConfigs() dynamic.ResourceInterface { + return e.Kube.Dynamic.Resource(atev1.GroupVersion.WithResource("sandboxconfigs")) +} + +// sandboxConfigName returns the name of the SandboxConfig of class rendered +// as unnamed, the config's manifest without its name and label. The name is +// substrate--. hash is the first 10 hex digits of the SHA-256 of +// unnamed without its comment lines, so builds that render the same config +// share it. Editing the config later does not rename it. +func sandboxConfigName(class atev1.SandboxClass, unnamed []byte) string { + h := sha256.New() + for _, line := range strings.SplitAfter(string(unnamed), "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "#") { + h.Write([]byte(line)) + } + } + return fmt.Sprintf("substrate-%s-%s", hex.EncodeToString(h.Sum(nil))[:10], class) +} + +// classDefaultWarning returns a warning about the class default of class, or +// "" for none. configs are the SandboxConfigs in the cluster, and name is +// this build's config. +// +// - It warns when the config has the class-default annotation but is not +// the class default, because a newer default exists. That happens when an +// earlier build's config is deployed again. +// - It warns when the class has no default, so pools of the class without a +// configRef resolve none. +func classDefaultWarning(configs []atev1.SandboxConfig, name string, class atev1.SandboxClass) string { + def := atev1.ClassDefaultSandboxConfig(configs, class) + if def == nil { + return fmt.Sprintf("no SandboxConfig of sandbox class %q has the %s annotation, so WorkerPools of the class without a configRef resolve no config", + class, atev1.SandboxConfigClassDefaultAnnotation) + } + i := slices.IndexFunc(configs, func(sc atev1.SandboxConfig) bool { return sc.Name == name }) + if i >= 0 && configs[i].Annotations[atev1.SandboxConfigClassDefaultAnnotation] == "true" && def.Name != name { + return fmt.Sprintf("SandboxConfig %s has the %s annotation but is not the %s class default, because %s is newer", + name, atev1.SandboxConfigClassDefaultAnnotation, class, def.Name) + } + return "" +} diff --git a/cmd/ate-setup/internal/steps/sandboxconfig_test.go b/cmd/ate-setup/internal/steps/sandboxconfig_test.go new file mode 100644 index 0000000000..e7d4b721f3 --- /dev/null +++ b/cmd/ate-setup/internal/steps/sandboxconfig_test.go @@ -0,0 +1,192 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" +) + +func TestSandboxConfigName(t *testing.T) { + tests := []struct { + name string + class atev1.SandboxClass + manifest string + want string + }{ + {name: "gvisor", class: atev1.SandboxClassGvisor, manifest: "spec: a\n", want: "substrate-d5ee99cf6b-gvisor"}, + {name: "comment lines do not count", class: atev1.SandboxClassGvisor, manifest: "# a\n # b\nspec: a\n", want: "substrate-d5ee99cf6b-gvisor"}, + {name: "another spec", class: atev1.SandboxClassGvisor, manifest: "spec: b\n", want: "substrate-22e34e826a-gvisor"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := sandboxConfigName(tt.class, []byte(tt.manifest)); got != tt.want { + t.Errorf("sandboxConfigName(%q, %q) = %q, want %q", tt.class, tt.manifest, got, tt.want) + } + }) + } +} + +func TestClassDefaultWarning(t *testing.T) { + const name = "substrate-v2-gvisor" + t0 := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + config := func(name string, class atev1.SandboxClass, created time.Time, isDefault, deleting bool) atev1.SandboxConfig { + sc := atev1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: name, CreationTimestamp: metav1.NewTime(created)}, + Spec: atev1.SandboxConfigSpec{SandboxClass: class}, + } + if isDefault { + sc.Annotations = map[string]string{atev1.SandboxConfigClassDefaultAnnotation: "true"} + } + if deleting { + sc.DeletionTimestamp = &metav1.Time{Time: created.Add(time.Hour)} + } + return sc + } + gvisor, microvm := atev1.SandboxClassGvisor, atev1.SandboxClassMicroVM + + tests := []struct { + name string + configs []atev1.SandboxConfig + // wantWarning is a substring of the warning, empty for none. + wantWarning string + }{ + { + name: "fresh cluster", + wantWarning: "no SandboxConfig", + }, + { + name: "the only config, annotated", + configs: []atev1.SandboxConfig{config(name, gvisor, t0, true, false)}, + }, + { + name: "the only config, not annotated", + configs: []atev1.SandboxConfig{config(name, gvisor, t0, false, false)}, + wantWarning: "no SandboxConfig", + }, + { + name: "not annotated with an existing default", + configs: []atev1.SandboxConfig{ + config("substrate-v1-gvisor", gvisor, t0, true, false), + config(name, gvisor, t0.Add(time.Hour), false, false), + }, + }, + { + name: "not annotated when only another class has a default", + configs: []atev1.SandboxConfig{ + config("substrate-v1-microvm", microvm, t0, true, false), + config(name, gvisor, t0.Add(time.Hour), false, false), + }, + wantWarning: "no SandboxConfig", + }, + { + name: "not annotated when the only default is being deleted", + configs: []atev1.SandboxConfig{ + config("substrate-v1-gvisor", gvisor, t0, true, true), + config(name, gvisor, t0.Add(time.Hour), false, false), + }, + wantWarning: "no SandboxConfig", + }, + { + name: "annotated over an older default", + configs: []atev1.SandboxConfig{ + config("substrate-v1-gvisor", gvisor, t0, true, false), + config(name, gvisor, t0.Add(time.Hour), true, false), + }, + }, + { + name: "annotated, older than the newest of two newer defaults", + configs: []atev1.SandboxConfig{ + config(name, gvisor, t0, true, false), + config("substrate-v3-gvisor", gvisor, t0.Add(time.Hour), true, false), + config("substrate-v4-gvisor", gvisor, t0.Add(2*time.Hour), true, false), + }, + wantWarning: "because substrate-v4-gvisor is newer", + }, + { + name: "annotated, created in the same second as a default with a smaller name", + configs: []atev1.SandboxConfig{ + config("substrate-v1-gvisor", gvisor, t0, true, false), + config(name, gvisor, t0, true, false), + }, + wantWarning: "because substrate-v1-gvisor is newer", + }, + { + name: "not annotated, older than a newer default", + configs: []atev1.SandboxConfig{ + config(name, gvisor, t0, false, false), + config("substrate-v3-gvisor", gvisor, t0.Add(time.Hour), true, false), + }, + }, + { + name: "annotated, older than a newer default being deleted", + configs: []atev1.SandboxConfig{ + config(name, gvisor, t0, true, false), + config("substrate-v3-gvisor", gvisor, t0.Add(time.Hour), true, true), + }, + }, + { + name: "annotated, older than a newer default of another class", + configs: []atev1.SandboxConfig{ + config(name, gvisor, t0, true, false), + config("substrate-v3-microvm", microvm, t0.Add(time.Hour), true, false), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + warning := classDefaultWarning(tt.configs, name, gvisor) + if (warning == "") != (tt.wantWarning == "") || !strings.Contains(warning, tt.wantWarning) { + t.Errorf("classDefaultWarning() = %q, want it to contain %q", warning, tt.wantWarning) + } + }) + } +} + +func TestRequireSandboxConfigNotDeleting(t *testing.T) { + const name = "substrate-v2-gvisor" + config := func(name string, deleting bool) atev1.SandboxConfig { + sc := atev1.SandboxConfig{ObjectMeta: metav1.ObjectMeta{Name: name}} + if deleting { + sc.DeletionTimestamp = &metav1.Time{Time: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)} + } + return sc + } + + tests := []struct { + name string + configs []atev1.SandboxConfig + wantErr bool + }{ + {name: "fresh cluster"}, + {name: "only other configs", configs: []atev1.SandboxConfig{config("substrate-v1-gvisor", false)}}, + {name: "exists", configs: []atev1.SandboxConfig{config(name, false)}}, + {name: "being deleted", configs: []atev1.SandboxConfig{config(name, true)}, wantErr: true}, + {name: "another config being deleted", configs: []atev1.SandboxConfig{config("substrate-v1-gvisor", true)}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := requireSandboxConfigNotDeleting(tt.configs, name) + if (err != nil) != tt.wantErr { + t.Fatalf("requireSandboxConfigNotDeleting() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig.go b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig.go index 6227f4585d..af87fdf25a 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig.go +++ b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig.go @@ -31,9 +31,8 @@ import ( // // - With configRef, that config is used. It must exist, belong to the same // class, and not be being deleted. -// - Without configRef, the newest config of the class that is annotated as -// the class default and is not being deleted is used. If two were created -// at the same time, the name that sorts first wins. +// - Without configRef, the class default from +// atev1.ClassDefaultSandboxConfig is used. func resolveLatestSandboxConfig(entry atev1.WorkerPoolSandboxClass, configs []atev1.SandboxConfig) (*atev1.SandboxConfig, string, string) { if entry.ConfigRef != nil { name := entry.ConfigRef.Name @@ -54,22 +53,13 @@ func resolveLatestSandboxConfig(entry atev1.WorkerPoolSandboxClass, configs []at return sc, "", "" } - isDefault := func(sc atev1.SandboxConfig) bool { - return sc.Spec.SandboxClass == entry.Name && - sc.Annotations[atev1.SandboxConfigClassDefaultAnnotation] == "true" - } - var best *atev1.SandboxConfig - for i := range configs { - sc := &configs[i] - if isDefault(*sc) && sc.DeletionTimestamp.IsZero() && (best == nil || newerSandboxConfig(sc, best)) { - best = sc - } - } - if best != nil { - return best, "", "" + if sc := atev1.ClassDefaultSandboxConfig(configs, entry.Name); sc != nil { + return sc, "", "" } isDeletingDefault := func(sc atev1.SandboxConfig) bool { - return isDefault(sc) && !sc.DeletionTimestamp.IsZero() + return sc.Spec.SandboxClass == entry.Name && + sc.Annotations[atev1.SandboxConfigClassDefaultAnnotation] == "true" && + !sc.DeletionTimestamp.IsZero() } if slices.ContainsFunc(configs, isDeletingDefault) { return nil, atev1.WorkerPoolReasonSandboxConfigDeleting, @@ -122,15 +112,6 @@ func keepSandboxConfig(class atev1.SandboxClass, inUse string, configs []atev1.S } } -// newerSandboxConfig reports whether a takes precedence over b as a class -// default. -func newerSandboxConfig(a, b *atev1.SandboxConfig) bool { - if !a.CreationTimestamp.Equal(&b.CreationTimestamp) { - return b.CreationTimestamp.Before(&a.CreationTimestamp) - } - return a.Name < b.Name -} - // resolveSandboxClasses resolves every entry of wp.Spec.SandboxClasses. It // returns one status entry per class, and whether every class resolved. Each // entry carries the config the class uses, nil if it has none, and its diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index d609850dbf..78e109f177 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -17,9 +17,8 @@ # suspend/resume across pods shows the count continuing — proving the guest # memory snapshot round-tripped. The ActorTemplate is not a CRD: it lives in # counter-microvm-template.yaml.tmpl and is created through the ate API with -# `kubectl ate create actor-template`. The cluster-wide `microvm` -# SandboxConfig referenced by the pool is installed by -# hack/install-microvm-deps.sh +# `kubectl ate create actor-template`. The pool follows the microVM class +# default SandboxConfig, which hack/install-microvm-deps.sh installs. apiVersion: v1 kind: Namespace @@ -41,8 +40,6 @@ spec: replicas: 2 sandboxClasses: - name: microvm - configRef: - name: microvm workerImage: ko://go.ate.dev/substrate/cmd/ateworker-microvm template: # Per-worker resources: these size the worker POD and advertise its diff --git a/demos/counter/counter.yaml.tmpl b/demos/counter/counter.yaml.tmpl index 190ec65e19..a2c0b93637 100644 --- a/demos/counter/counter.yaml.tmpl +++ b/demos/counter/counter.yaml.tmpl @@ -40,7 +40,8 @@ spec: workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor # No configRef: the pool follows the class default, the newest SandboxConfig # of the class annotated sandboxconfig.ate.dev/is-class-default=true - # (gvisor-default as installed), and moves to a new default when one appears. + # (the one ate-setup installs), and moves to a new default when + # one appears. sandboxClasses: - name: gvisor template: diff --git a/demos/egress/egress-microvm.yaml.tmpl b/demos/egress/egress-microvm.yaml.tmpl index d4f589b596..e1ac4ead12 100644 --- a/demos/egress/egress-microvm.yaml.tmpl +++ b/demos/egress/egress-microvm.yaml.tmpl @@ -16,9 +16,9 @@ # egress demo, so the networking suite's egress tests run against both # sandbox classes. The ActorTemplate is a substrate resource, not a CRD: it # lives in egress-microvm-template.yaml.tmpl and is created through the -# ate API with `kubectl ate create actor-template`. The cluster-wide `microvm` -# SandboxConfig referenced by the pool is installed by -# hack/install-microvm-deps.sh +# ate API with `kubectl ate create actor-template`. The pool follows the +# microVM class default SandboxConfig, which hack/install-microvm-deps.sh +# installs. apiVersion: v1 kind: Namespace diff --git a/docs/api-guide.md b/docs/api-guide.md index 7fba29d68a..c17fc9ab6f 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -61,8 +61,9 @@ spec: workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor - configRef: - name: gvisor-default + # Optional. Pins the pool to one SandboxConfig instead of the class default. + # configRef: + # name: substrate-04e4d62011-gvisor template: labels: project: agent-platform @@ -520,9 +521,9 @@ Each entry of `versions` (`SandboxVersionConfig`): | :--- | :--- | :--- | | `name` | `string` | **Required.** Identifies the version within the config. A DNS label (lower-case alphanumerics and `-`, at most 63 characters). | | `state` | `string` | Optional. `Enabled` (default) or `Disabled`. The version named by `defaultVersion` cannot be `Disabled`. | -| `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A microVM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, is unreachable, or fails with an error it does not recognize, such as GCS refusing its credentials for a public bucket) is then read anonymously, which serves public buckets such as `gs://gvisor`; any other refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (a snapshot location followed by `/atespaces/`), so assets must not be stored there. On GKE, setup-gcp lets the plugin read only the asset bucket, `-assets`, so keep assets there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | +| `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A microVM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. On update it checks only versions that are new or whose assets changed. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, is unreachable, or fails with an error it does not recognize, such as GCS refusing its credentials for a public bucket) is then read anonymously, which serves public buckets such as `gs://gvisor`; any other refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (a snapshot location followed by `/atespaces/`), so assets must not be stored there. On GKE, setup-gcp lets the plugin read only the asset bucket, `-assets`, so keep assets there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | -A cluster-wide gVisor `SandboxConfig` (`gvisor-default`, annotated as the class default) is installed with the platform, so gVisor `WorkerPool`s use it automatically without any extra setup. +Substrate installs a gVisor `SandboxConfig` rendered from [`manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl`](../manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl). It is named `substrate--gvisor`, hashed from the rendered config, so builds and releases with the same config share one. Editing the config does not rename it. The label `sandboxconfig.ate.dev/created-by-version` records the Substrate version that created it. `ate-setup deploy ate-system` and `ate-setup deploy sandboxconfig` add the class-default annotation when they create it, unless run with `--create-with-default-annotation=false`, so gVisor `WorkerPool`s use it without any extra setup. They leave an existing config as it is, annotations included, so use `kubectl annotate` to change its annotation. If a newer config of the class already has the annotation, the newer one stays the default, and `ate-setup` warns. Configs from earlier builds stay in the cluster. ### Class default @@ -544,7 +545,7 @@ atecontroller keeps a `SandboxConfig` from being deleted while a `WorkerPool` us A delete cannot be undone, and an object that is being deleted cannot be recreated under the same name. Applying its manifest again only updates the terminating object. To recover from deleting a class default that pools still use: 1. Create a replacement under a different name with the `is-class-default` annotation. Pools move to it on their own, the finalizer is released, and the old config disappears. -2. Once the old config is gone, the next `ate-setup` upgrade recreates the shipped config under its original name. Pools move back to it, because it is then the newest default. +2. Once the old config is gone, `ate-setup deploy sandboxconfig` recreates this build's config under its original name. Pools move back to it, because it is then the newest default. atecontroller alone adds and releases the finalizer. While it is down, `kubectl delete sandboxconfig` of a config that carries the finalizer hangs until atecontroller is back. `ate-setup` strips the finalizer from every config when it deletes the control plane. @@ -554,7 +555,9 @@ atecontroller alone adds and releases the finalizer. While it is down, `kubectl apiVersion: ate.dev/v1 kind: SandboxConfig metadata: - name: gvisor-default + name: substrate-04e4d62011-gvisor + labels: + sandboxconfig.ate.dev/created-by-version: "v0.4.2" annotations: sandboxconfig.ate.dev/is-class-default: "true" spec: @@ -578,7 +581,7 @@ spec: A `microvm` `SandboxConfig` supplies the [Kata Containers](https://katacontainers.io/) + [Cloud Hypervisor](https://www.cloudhypervisor.org/) toolchain instead of `runsc`. Each architecture of each version must define the full asset set — `cloud-hypervisor`, `virtiofsd`, `kata-kernel`, and `kata-image` — which a `ValidatingAdmissionPolicy` enforces at apply time. Worker pods for a microVM pool require `/dev/kvm` and nested-virtualization-capable nodes. The controller requests those devices on the pod automatically, and atelet advertises them only where they exist, so placement follows the hardware rather than a node label. Clusters that reserve nested-virt nodes with an `ate.dev/sandboxClass=microvm` taint are still tolerated: advertising a device attracts these pods to capable nodes but repels nothing else from them. The same convention applies to every class: worker pods of a pool tolerate `ate.dev/sandboxClass=:NoSchedule`, so a cluster can reserve a node pool per sandbox class with that taint, and the atelet DaemonSet tolerates the key for any value. On GKE, a pool can also select a ComputeClass with `template.nodeSelector` (`cloud.google.com/compute-class: `): GKE taints the nodes it creates for the class `cloud.google.com/compute-class=:NoSchedule` and adds the matching toleration to the worker pods, and the atelet DaemonSet tolerates that key as well. -See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. +`hack/install-microvm-deps.sh --install` stages the assets and creates `substrate--microvm` from [`manifests/microvm/sandboxconfig-microvm.yaml.tmpl`](../manifests/microvm/sandboxconfig-microvm.yaml.tmpl), hashed from the rendered config and labeled with `VERSION`, with the class-default annotation unless run with `--create-with-default-annotation=false`. Like the gVisor installers, it leaves an existing config as it is. See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. --- diff --git a/docs/dev/microvm-local.md b/docs/dev/microvm-local.md index 66aba29851..545773f301 100644 --- a/docs/dev/microvm-local.md +++ b/docs/dev/microvm-local.md @@ -60,7 +60,7 @@ This is a one-shot bring-up: it deploys the control plane, installs the cluster-wide microVM deps via `hack/install-microvm-deps.sh` — assembling the guest runtime assets for your architecture (skipped if already present under `bin/microvm-assets/`), staging them into the in-cluster rustfs bucket, and -applying the `microvm` `SandboxConfig` — then deploys the demo worker pool + +creating the microVM class default `SandboxConfig` — then deploys the demo worker pool + template. ### 4. Verify diff --git a/docs/sandboxconfig-upgrade.md b/docs/sandboxconfig-upgrade.md new file mode 100644 index 0000000000..d79b62f4da --- /dev/null +++ b/docs/sandboxconfig-upgrade.md @@ -0,0 +1,122 @@ +# SandboxConfig upgrade + +This guide covers the SandboxConfig part of an upgrade: step 4d of the [upgrade runbook](upgrade.md), and its rollback. Run it from the runbook's shell, with `NEW_RELEASE` set and `~/ate-upgrade` in place. + +Each sandbox class has a SandboxConfig rendered from a template in the repository, named `substrate--gvisor` or `substrate--microvm` after a hash of the rendered config. A release that changes a template creates a new config, labeled `sandboxconfig.ate.dev/created-by-version=`. A release that does not reuses the existing one. Configs from earlier releases stay in the cluster. A WorkerPool without a `configRef` uses the newest config of its class with the `sandboxconfig.ate.dev/is-class-default` annotation. A pool with a `configRef` uses that config. [Class default](api-guide.md#class-default) in the API guide has the full rules. Moving a pool to another config does not roll its workers. + +New actors start on the default version of their pool's config. A suspended actor restores its memory only on a version with the same assets as the one its snapshot was taken on. If the pool's config has no such version, the actor resumes without its memory, keeping its files. The [version skew policy](version-skew-policy.md#sandboxconfig) says which versions a release keeps. + +## Upgrade + +### gVisor + +```bash +go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml +``` + +This applies the SandboxConfig admission policy and creates the new release's gVisor config as the class default, unless a config with the same hash exists. It prints the config's name. gVisor pools without a `configRef` move to it. Pools pinned to a config with `configRef` stay on it until you change their `configRef`. + +### microVM + +If the cluster runs microVM pools, also run this from the new checkout: + +```bash +BUCKET_NAME=$(sed -n 's/^ *bucketName: //p' ~/ate-upgrade/new.yaml) VERSION=$NEW_RELEASE hack/install-microvm-deps.sh --install +``` + +The script reads neither `new.yaml` nor the image tag, so the variables give it the bucket and the release for the label. It creates the new release's microVM config the same way as the gVisor config. It first stages the assets at fixed paths in the bucket. If the release changes them, earlier microVM configs no longer match the bucket, and atelet cannot fetch their assets on nodes that have not cached them. MicroVM pools pinned to an earlier config, or left on one by `--create-with-default-annotation=false`, then need to move to the new config. + +### Create the configs without moving pools + +Add `--create-with-default-annotation=false` to both commands above. Later runs leave the configs as they are, including `deploy ate-system` in step 4e of the runbook. Later, annotate each config they created, using the name they printed: + +```bash +kubectl annotate sandboxconfig sandboxconfig.ate.dev/is-class-default=true +``` + +### Check + +Each pool's `IN-USE` column shows the config it uses. `PINNED` is the config its `configRef` names, and `PENDING` is a config it is switching to. + +```bash +kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,PINNED:.spec.sandboxClasses[*].configRef.name,IN-USE:.status.sandboxClasses[*].configRef.name,PENDING:.status.sandboxClasses[*].pendingConfigRef.name' +``` + +## Delete earlier configs + +Once the upgrade is done, delete the earlier configs that no pool uses, so they do not pile up. Check first that no pool names the config in any column of [Check](#check). A config that a pool uses is not deleted until no pool does. If you roll back later, [Roll back](#roll-back) creates the old release's config again. + +```bash +kubectl delete sandboxconfig +``` + +## Roll back + +Rollback runs from the old checkout, in two parts around the components in the runbook's [Roll back step 4](upgrade.md#roll-back-step-4). + +### Move pools off the new configs + +The new configs are the ones the upgrade created, labeled with `NEW_RELEASE`. If the release did not change a template, there are none. List them: + +```bash +kubectl get sandboxconfigs -l sandboxconfig.ate.dev/created-by-version=$NEW_RELEASE +``` + +List the pools as in [Check](#check). Change or remove the `configRef` of each pool whose `PINNED` column names a new config. A config is not deleted while a pool uses it, so a pinned pool blocks the delete below. + +List the configs and their class-default annotations: + +```bash +kubectl get sandboxconfigs -o custom-columns='NAME:.metadata.name,CLASS:.spec.sandboxClass,CLASS-DEFAULT:.metadata.annotations.sandboxconfig\.ate\.dev/is-class-default,CREATED:.metadata.creationTimestamp' +``` + +Each class needs a config outside the new ones with `CLASS-DEFAULT` set to `true`, usually the old release's config. Otherwise pools without a `configRef` stay on the new config, and the delete below does not finish. + +If a class has none and the list has the old release's config of that class, annotate that config: + +```bash +kubectl annotate sandboxconfig sandboxconfig.ate.dev/is-class-default=true +``` + +If the old release's gVisor config is gone, create it. It gets the annotation: + +```bash +go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/old.yaml +``` + +This also applies the old admission policy before the new config is gone. If the delete then does not finish, see [Restore the old admission policy](#restore-the-old-admission-policy). + +If the cluster runs microVM pools, run this. It stages the old release's microVM assets, and creates the old release's microVM config with the annotation if it is gone. + +```bash +BUCKET_NAME=$(sed -n 's/^ *bucketName: //p' ~/ate-upgrade/old.yaml) VERSION=$OLD_RELEASE hack/install-microvm-deps.sh --install +``` + +Then delete the new configs: + +```bash +kubectl delete sandboxconfigs -l sandboxconfig.ate.dev/created-by-version=$NEW_RELEASE --wait=false +``` + +A config being deleted is never the class default, so pools without a `configRef` move to the newest other config with the annotation. ate-controller finishes the delete once no pool uses the config. `--wait=false` returns before that, because if the new ate-controller is broken, the delete only finishes after the runbook restores the old one. + +If the old release's config does not list the version an actor was suspended on, the actor resumes without its memory, keeping its files. + +### Restore the old admission policy + +Run this after `deploy ate-controller` in the runbook's [Roll back step 4](upgrade.md#roll-back-step-4). Wait for the new configs to be gone, then apply the old release's policy. + +```bash +kubectl wait --for=delete sandboxconfigs -l sandboxconfig.ate.dev/created-by-version=$NEW_RELEASE --timeout=5m +kubectl apply --server-side --force-conflicts --field-manager=ate-setup -f manifests/ate-install/sandboxconfig-validation.yaml +``` + +ate-controller finishes a delete by removing the config's finalizer, which is an update. The new policy checks only versions that an update adds or whose assets it changes. An older policy may check them, and reject the update if the new config fails its rules. So the wait comes first. + +If the wait times out and a pool still names a new config in [Check](#check), fix that pool. If no pool does, the old policy rejected the update, which can happen after `deploy sandboxconfig` above applied it early. Apply the new release's policy, then run the commands above again: + +```bash +kubectl apply --server-side --force-conflicts --field-manager=ate-setup -f ~/ate-upgrade/new/manifests/ate-install/sandboxconfig-validation.yaml +``` + +Apply the policy even if the upgrade created no config, because step 4d applies it first. diff --git a/docs/upgrade.md b/docs/upgrade.md index f440bc5540..8be0536b1b 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -218,11 +218,13 @@ kubectl -n ate-system rollout status deploy/atenet-egress go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml ``` -This applies the SandboxConfig admission policy and the default SandboxConfig. It does not roll workers. If the release changes a sandbox version, new actors start on the new version, and actors suspended on the previous one still restore on it. +By default this creates the new release's gVisor SandboxConfig as the gVisor class default, and gVisor pools without a `configRef` move to it without rolling their workers. If the release did not change the config, it exists already and nothing changes. If the cluster runs microVM pools, also follow [microVM](sandboxconfig-upgrade.md#microvm) in the SandboxConfig upgrade guide. -- **Done:** the command exits 0. -- **Stuck:** the command fails. [Roll back step 4](#roll-back-step-4). -- **Actors:** a suspended actor whose sandbox version the release drops resumes without its memory, keeping its files. +To control which pools move and when, and for what happens to suspended actors, see the [SandboxConfig upgrade guide](sandboxconfig-upgrade.md). + +- **Done:** the commands exit 0. +- **Stuck:** a command fails. [Roll back step 4](#roll-back-step-4). +- **Actors:** if the new config does not list the sandbox version an actor was suspended on, the actor resumes without its memory, keeping its files. #### 4e. The rest of the system bundle @@ -238,7 +240,7 @@ This applies what the steps above do not, such as the ate-system NetworkPolicies Then install the new `kubectl ate` with `go install ./cmd/kubectl-ate`. Actor owners can now use API fields and commands new in this release, and you can set WorkerPool, SandboxConfig and CSIDriverConfig fields new in this release. -The upgrade is done when step 4 is done and [Where am I?](#where-am-i) shows every component and pool moved, every pool done rolling and none paused. Keep `~/ate-upgrade` while you might still roll back, then delete it. +The upgrade is done when step 4 is done and [Where am I?](#where-am-i) shows every component and pool moved, every pool done rolling and none paused. Then delete the earlier SandboxConfigs that no pool uses, as in [Delete earlier configs](sandboxconfig-upgrade.md#delete-earlier-configs). Keep `~/ate-upgrade` while you might still roll back, then delete it. ## Something's wrong, or I need to roll back @@ -265,17 +267,18 @@ cd ~/ate-upgrade/old #### Roll back step 4 -Tell actor owners to stop using API fields new in this release, because the old API server rejects them. Then: +Tell actor owners to stop using API fields new in this release, because the old API server rejects them. Follow [Move pools off the new configs](sandboxconfig-upgrade.md#move-pools-off-the-new-configs) in the SandboxConfig upgrade guide. Then: ```bash -go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/old.yaml go run ./cmd/ate-setup deploy atenet --config ~/ate-upgrade/old.yaml go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/old.yaml go run ./cmd/ate-setup deploy apiserver --config ~/ate-upgrade/old.yaml go install ./cmd/kubectl-ate ``` -`deploy sandboxconfig` removes a sandbox version new in this release, so actors suspended on it resume without their memory. `deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. No pool rolls. +Then follow [Restore the old admission policy](sandboxconfig-upgrade.md#restore-the-old-admission-policy). + +`deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. No pool rolls. Fields that the new release added to WorkerPools, SandboxConfigs or CSIDriverConfigs are dropped from those objects. Remove them from your YAML files too, or `kubectl apply` rejects the files. diff --git a/docs/version-skew-policy.md b/docs/version-skew-policy.md index 64427c1b7a..68e7798f2a 100644 --- a/docs/version-skew-policy.md +++ b/docs/version-skew-policy.md @@ -78,13 +78,13 @@ Between compatible releases, a CRD change only adds optional fields. The CRDs se ### SandboxConfig -Each sandbox class has a default SandboxConfig, such as `gvisor-default`, that lists the sandbox versions a cluster may run. A release that changes a sandbox version adds the new one, points `defaultVersion` at it, and keeps the previous release's version unless that version has a known vulnerability. +Each sandbox class has a SandboxConfig rendered from a template in the repository and named after a hash of the rendered config, such as `substrate-04e4d62011-gvisor`. The config lists the sandbox versions a cluster may run. A release that changes the template creates a new config. By default the upgrade annotates it as the class default, and pools without a `configRef` move to it. The previous release's config stays in the cluster. A release that changes a sandbox version adds the new one to the template, points `defaultVersion` at it, and keeps the previous release's version unless that version has a known vulnerability. -A kept version's assets must stay fetchable, so a release does not overwrite them. The upgrade applies these lists. That does not change the pod template, so workers do not roll. +A kept version's assets must stay fetchable, so a release does not overwrite them. The microVM assets live at fixed paths in the cluster's bucket, so a microVM release that changes them replaces the version instead. Moving a pool to a new config does not change the pod template, so workers do not roll. -An actor that starts without memory state, new or cold booting, gets the default version. An actor restoring memory state gets the version its snapshot was taken with, and stays on it until it next starts without memory state. If that version is gone, it cold boots, keeping its files and losing its memory. Rollback applies the N lists again, which removes the N+1 versions. +An actor that starts without memory state, new or cold booting, gets the default version. An actor restoring memory state gets a version with the same assets as the one its snapshot was taken on, and stays on it until it next starts without memory state. If the pool's config has no such version, the actor cold boots, keeping its files and losing its memory. Rollback deletes the configs N+1 created, if any, so the newest other config with the class-default annotation, usually N's, is the class default again. That config does not list the sandbox versions new in N+1. An actor suspended on one of them resumes without its memory. - +The [SandboxConfig upgrade guide](sandboxconfig-upgrade.md) covers the details. ### Database schema @@ -124,7 +124,7 @@ To move from N to N+1, in this order, waiting for each step to finish: 8. The rest of the substrate system bundle: what no step above deploys. 9. Clients. -Rollback from N+1 to N is the same list in reverse, with release N, except that the schema migration and step 8 are not undone. Clients stop sending N+1 fields before `ate-api-server` goes back, and `atelet` goes back only after every worker is back at N. Rolling back further than N is not supported. +Rollback from N+1 to N is the same list in reverse, with release N, except that the schema migration and step 8 are not undone. Step 7 deletes the N+1 SandboxConfigs before step 5, and restores the N admission policy after the deletes finish. See [Roll back](sandboxconfig-upgrade.md#roll-back) in the SandboxConfig upgrade guide. Clients stop sending N+1 fields before `ate-api-server` goes back, and `atelet` goes back only after every worker is back at N. Rolling back further than N is not supported. ### Skipping releases diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index 312da2e346..995b0575b8 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -14,26 +14,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Install (or delete) the cluster-wide micro-VM (kata + cloud-hypervisor) -# dependencies. This is opt-in on top of ate-setup deploy ate-system, -# which only installs the default gVisor SandboxConfig. Used by: +# Install the cluster-wide microVM (kata + cloud-hypervisor) dependencies. +# This is opt-in on top of ate-setup deploy ate-system, which only installs +# the gVisor SandboxConfig. Used by: # * hack/run-microvm-demo.sh (before applying the counter-microvm demo) # * benchmarking/automation/orchestrator.py (before deploying a microvm # benchmark workload) # # On --install: assembles the asset set (assemble.sh; skipped if OUT already # has them), stages the assets under kata-assets/ to the cluster's object -# store bucket (rustfs on kind, GCS on GKE), and applies the cluster-wide -# `microvm` SandboxConfig referencing those assets. Every asset sha256 is -# pinned in the manifest, so the apply only substitutes the bucket name. +# store bucket (rustfs on kind, GCS on GKE), and creates this build's +# microVM SandboxConfig from the template unless it exists. It is named as +# `ate-setup` names the gVisor config, substrate--microvm, and labeled +# with VERSION. Every asset sha256 is pinned in the template, so the render +# only substitutes the name, the label and the bucket. A config this run +# creates gets the sandboxconfig.ate.dev/is-class-default annotation, unless +# --create-with-default-annotation=false. An existing config is left as it +# is, annotations included. # -# On --delete: removes the SandboxConfig from the cluster. The delete is best -# effort. A SandboxConfig that a WorkerPool uses carries a protection -# finalizer, which atecontroller releases only once no pool's status names -# the config. So the script leaves the config in place if any WorkerPool -# still uses it. Otherwise it waits a bounded time for the finalizer to go. -# Bucket contents are left alone (they're inert until a SandboxConfig points -# at them, and re-staging is cheap on next install). +# Nothing here deletes the configs. Like the gVisor configs, they stay until +# the SandboxConfig CRD is deleted with the control plane. # # Like the other hack scripts, this sources .ate-dev-env.sh for the cluster / # registry / bucket settings unless NO_DEV_ENV is set. @@ -49,6 +49,8 @@ # OUT asset dir (default: $PWD/bin/microvm-assets/$ARCH, gitignored). # ATE_INSTALL_KIND "true" for the kind path (stage assets to rustfs); default # false uploads assets to GCS. +# VERSION (optional) Substrate version the SandboxConfig is labeled +# with (default: git describe). set -o errexit -o nounset -o pipefail @@ -67,22 +69,26 @@ ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" usage() { cat <&2; usage; exit 1 ;; esac @@ -129,41 +135,79 @@ else ASSET_BUCKET_NAME="${BUCKET_NAME}-assets" fi -if [[ "${action}" == "delete" ]]; then - # A WorkerPool that uses the config holds it. The delete would only mark - # the config Terminating, and `kubectl delete` would wait for it without - # end. A Terminating config also cannot be recreated under its name. So no - # delete is issued at all. The config is left in place and the pools are - # named instead. A cluster without the WorkerPool CRD has no users. - users="$(run_kubectl get workerpools --all-namespaces \ - -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name} {.status.sandboxClasses[*].configRef.name}{"\n"}{end}' 2>/dev/null \ - | awk '{ for (i = 2; i <= NF; i++) if ($i == "microvm") { print $1; break } }' || true)" - if [[ -n "${users}" ]]; then - echo "Warning: the microvm SandboxConfig is left in place; these WorkerPools" >&2 - echo " still use it:" >&2 - while IFS= read -r pool; do echo " ${pool}" >&2; done <<<"${users}" - echo " Delete them, or point their spec.sandboxClasses[].configRef at" >&2 - echo " another microvm SandboxConfig, and run this again." >&2 - exit 0 - fi - log "Deleting microvm SandboxConfig..." - # Delete by name rather than by manifest so a missing/edited template file - # doesn't block cleanup on an older cluster. atecontroller releases the - # protection finalizer within a reconcile. The timeout turns a missing - # atecontroller into an error instead of a hang. - if ! run_kubectl delete --ignore-not-found --timeout=60s sandboxconfig microvm; then - echo "Error: the microvm SandboxConfig did not finish deleting. It stays" >&2 - echo " Terminating until atecontroller releases its protection" >&2 - echo " finalizer; check that atecontroller is running, then run this" >&2 - echo " again." >&2 - exit 1 - fi - log "Done. (Bucket assets at ${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/ left in place.)" - exit 0 -fi +DEFAULT_ANNOTATION="sandboxconfig.ate.dev/is-class-default" + +# Prints one line per microVM SandboxConfig: +# name|creationTimestamp|default annotation|deletionTimestamp. +get_microvm_configs() { + run_kubectl get sandboxconfigs \ + -o jsonpath='{range .items[?(@.spec.sandboxClass=="microvm")]}{.metadata.name}{"|"}{.metadata.creationTimestamp}{"|"}{.metadata.annotations.sandboxconfig\.ate\.dev/is-class-default}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' +} + +# This fails before the slow assemble and stage if the control plane, and +# with it the SandboxConfig CRD, is not installed. +microvm_configs="$(get_microvm_configs)" # --- install ---------------------------------------------------------------- +# Name this build's config the way `ate-setup` names the gVisor one (see +# sandboxConfigName in cmd/ate-setup/internal/steps/sandboxconfig.go): +# substrate--microvm, hashed from the config rendered without its name, +# label and comment lines. The version is checked here so a bad label value +# fails before the stage overwrites the bucket's assets. +manifest="$(sed -e "s|\${BUCKET_SCHEME}|${BUCKET_SCHEME}|g" \ + -e "s|\${ASSET_BUCKET_NAME}|${ASSET_BUCKET_NAME}|g" \ + "${MANIFEST_TEMPLATE}")" +SANDBOXCONFIG_NAME="substrate-$(sed '/^[[:space:]]*#/d' <<<"${manifest}" | shasum -a 256 | cut -c1-10)-microvm" +version="${VERSION:-$(git describe --tags --always --dirty 2>/dev/null || true)}" +if [[ ! "${version}" =~ ^([A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?)?$ || ${#version} -gt 63 ]]; then + echo "Error: version '${version}' is not a valid SandboxConfig label value; set VERSION to one." >&2 + exit 1 +fi + +# check_configs checks this build's config against the microVM configs in +# $1, as `ate-setup deploy sandboxconfig` checks the gVisor one. It sets +# exists when the config is in $1. +# * it exits when the config is being deleted +# * it sets warning when the config has the annotation but a newer config +# has it too. The newest one is the class default, so this one is not. +# * it sets warning when no config of the class has the annotation +# Names compare byte by byte, as atecontroller compares them. +check_configs() { + local name created is_default deleting + local existing_default="" newest_name="" newest_created="" + exists=false + while IFS='|' read -r name created is_default deleting; do + [[ -n "${name}" ]] || continue + if [[ "${name}" == "${SANDBOXCONFIG_NAME}" ]]; then + exists=true + existing_default="${is_default}" + if [[ -n "${deleting}" ]]; then + echo "Error: SandboxConfig ${SANDBOXCONFIG_NAME} is being deleted; wait until it" >&2 + echo " is gone, then run this again." >&2 + exit 1 + fi + fi + [[ "${is_default}" == "true" && -z "${deleting}" ]] || continue + # RFC 3339 timestamps in UTC sort as strings. + if [[ -z "${newest_name}" || "${created}" > "${newest_created}" ]] || + [[ "${created}" == "${newest_created}" && "${name}" < "${newest_name}" ]]; then + newest_name="${name}" + newest_created="${created}" + fi + done <<<"$1" + warning="" + if [[ -z "${newest_name}" ]]; then + warning="no microVM SandboxConfig has the ${DEFAULT_ANNOTATION} annotation, so microVM WorkerPools without a configRef resolve no config." + elif [[ "${existing_default}" == "true" && "${newest_name}" != "${SANDBOXCONFIG_NAME}" ]]; then + warning="SandboxConfig ${SANDBOXCONFIG_NAME} has the ${DEFAULT_ANNOTATION} annotation but is not the microVM class default, because ${newest_name} is newer." + fi +} +export LC_ALL=C +# Checked before the slow assemble and stage. The warning is about the +# cluster before the apply. +check_configs "${microvm_configs}" + # Target arch: match the images' platform (KO_DEFAULTPLATFORMS is set by # .ate-dev-env.sh on GKE and by the kind wrapper); fall back to the host arch. if [[ -z "${ARCH:-}" ]]; then @@ -215,14 +259,28 @@ else OUT="${OUT}" BUCKET="${ASSET_BUCKET_NAME}" hack/microvm-assets/stage-to-gcs.sh fi -# --- 3. apply the cluster-wide microvm SandboxConfig ----------------------- +# --- 3. create this build's microVM SandboxConfig --------------------------- # Every asset is downloaded rather than built, so all four carry committed, -# reproducible per-arch shas and the bucket's scheme and name are the only -# substitutions left. -log "Applying microvm SandboxConfig from ${MANIFEST_TEMPLATE}..." -sed -e "s|\${BUCKET_SCHEME}|${BUCKET_SCHEME}|g" \ - -e "s|\${ASSET_BUCKET_NAME}|${ASSET_BUCKET_NAME}|g" \ - "${MANIFEST_TEMPLATE}" \ - | run_kubectl apply -f - - -log "Done. Installed cluster-wide microvm SandboxConfig." +# reproducible per-arch shas and the name, the label, and the bucket's scheme +# and name are the only substitutions. An existing config may have been edited since, so it +# is left as it is. +if [[ "${exists}" == "true" ]]; then + log "SandboxConfig ${SANDBOXCONFIG_NAME} exists; leaving it and its annotations as they are." +else + log "Creating SandboxConfig ${SANDBOXCONFIG_NAME} from ${MANIFEST_TEMPLATE}..." + rendered="$(sed -e "s|\${SANDBOXCONFIG_NAME}|${SANDBOXCONFIG_NAME}|g" \ + -e "s|\${SUBSTRATE_VERSION}|${version}|g" <<<"${manifest}")" + # The annotation goes into the created object, so the config never exists + # without it. + if [[ "${create_with_default_annotation}" == "true" ]]; then + rendered="$(kubectl annotate --local -f - -o yaml "${DEFAULT_ANNOTATION}=true" <<<"${rendered}")" + fi + run_kubectl create -f - <<<"${rendered}" +fi +# Checked again against the config's real creation time and annotations. +microvm_configs="$(get_microvm_configs)" +check_configs "${microvm_configs}" +if [[ -n "${warning}" ]]; then + echo "Warning: ${warning}" >&2 +fi +log "Done. SandboxConfig ${SANDBOXCONFIG_NAME} is installed." diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 9ef7334c14..c667ed1bf6 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -20,8 +20,8 @@ # # Composes: # 1. ate-setup deploy ate-system (control plane) -# 2. hack/install-microvm-deps.sh --install (asset build/stage + cluster-wide -# microvm SandboxConfig) +# 2. hack/install-microvm-deps.sh --install (asset build/stage + the microVM +# class default SandboxConfig) # 3. Deploy the counter-microvm demo (worker pool manifest, atespace, and # ActorTemplate through the ate API). # diff --git a/internal/e2e/preflight.go b/internal/e2e/preflight.go index 62f7d04e21..32a913a95d 100644 --- a/internal/e2e/preflight.go +++ b/internal/e2e/preflight.go @@ -22,6 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "go.ate.dev/substrate/api/ateapi/v1" + atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" ) // PreflightChecks checks that the test environment is ready for the test suite. @@ -61,15 +62,19 @@ func PreflightChecks() error { return fmt.Errorf("ListActors RPC failed: %v", err) } - // The micro-VM class needs its SandboxConfig registered; without it every + // The microVM class needs a class default SandboxConfig; without it every // fixture in the run is misconfigured. // Note: Whether a node can actually host a micro-VM is left to the scheduler // and any issues are to be reported at runtime. Which device a sandbox class // requires should not be checked here to prevent drift between platforms. if IsMicroVM() { - if _, err := clients.SubstrateK8s.AteV1().SandboxConfigs().Get(ctx, SandboxClassMicroVM, metav1.GetOptions{}); err != nil { - return fmt.Errorf("E2E_SANDBOX_CLASS=%s but SandboxConfig/%s is missing (see docs/dev/microvm-local.md for cluster setup): %w", - SandboxClassMicroVM, SandboxClassMicroVM, err) + list, err := clients.SubstrateK8s.AteV1().SandboxConfigs().List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list SandboxConfigs: %w", err) + } + if atev1.ClassDefaultSandboxConfig(list.Items, atev1.SandboxClassMicroVM) == nil { + return fmt.Errorf("E2E_SANDBOX_CLASS=%s but no SandboxConfig of the class is annotated %s=true (see docs/dev/microvm-local.md for cluster setup)", + SandboxClassMicroVM, atev1.SandboxConfigClassDefaultAnnotation) } } diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl similarity index 59% rename from manifests/ate-install/sandboxconfig-gvisor.yaml rename to manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl index a444c303a6..800fedad68 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl @@ -12,23 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Cluster-wide SandboxConfig for the gVisor (runsc) sandbox class, shipped with -# the platform as the class default so gVisor WorkerPools resolve it unless -# they pin a specific configRef. atelet fetches the gVisor release tarball -# (gvisor.tar.zstd: runsc plus the gvisor-bin/ helpers runsc requires next to -# it) matching the worker node's architecture and extracts it locally. To pin a -# different release, add a version below and point defaultVersion at it, or -# create another SandboxConfig and reference it from the WorkerPool. +# Template for the gVisor (runsc) SandboxConfig. `ate-setup deploy +# sandboxconfig` and `deploy ate-system` create substrate--gvisor from +# it, hashed from the rendered config, and by default annotate it as the class +# default. They leave an existing config as it is, so editing this file +# changes only the configs that later builds create. # -# A bump adds a version and points defaultVersion at it. Keep the -# previous version unless it has a known vulnerability: an actor restores its -# memory only on the version its snapshot was taken with. +# atelet fetches the gVisor release tarball (gvisor.tar.zstd: runsc plus the +# gvisor-bin/ helpers runsc requires next to it) matching the worker node's +# architecture and extracts it locally. +# +# To change the sandbox, add a new version and point defaultVersion at it. +# Do not edit or remove the versions already listed. An actor restores its +# memory only on a version with the same assets as the one its snapshot was +# taken on, so dropping that version makes the actor cold boot and lose its +# memory. Remove a version only when it has a known vulnerability, or to make +# room under the limit of 16 versions. apiVersion: ate.dev/v1 kind: SandboxConfig metadata: - name: gvisor-default - annotations: - sandboxconfig.ate.dev/is-class-default: "true" + name: ${SANDBOXCONFIG_NAME} + labels: + sandboxconfig.ate.dev/created-by-version: "${SUBSTRATE_VERSION}" spec: sandboxClass: gvisor defaultVersion: v6 diff --git a/manifests/ate-install/sandboxconfig-validation.yaml b/manifests/ate-install/sandboxconfig-validation.yaml index ec28f4457f..ff44f7ddea 100644 --- a/manifests/ate-install/sandboxconfig-validation.yaml +++ b/manifests/ate-install/sandboxconfig-validation.yaml @@ -29,12 +29,24 @@ spec: apiVersions: ["v1"] operations: ["CREATE", "UPDATE"] resources: ["sandboxconfigs"] + # A later release can tighten this policy, and the configs of earlier + # releases stay in the cluster. Check only versions that are new or whose + # assets changed, so those configs can still be updated: removing a + # finalizer, disabling or deleting a version, or changing defaultVersion. + variables: + - name: changedVersions + expression: >- + object.spec.versions.filter(v, + oldObject == null || + oldObject.spec.sandboxClass != object.spec.sandboxClass || + !oldObject.spec.versions.exists(o, o.name == v.name && + has(o.assets) == has(v.assets) && (!has(v.assets) || o.assets == v.assets))) validations: # gVisor needs a "gvisor" release-tarball asset (gvisor.tar.zstd: runsc plus # its gvisor-bin/ helpers) for every architecture it advertises. - expression: >- object.spec.sandboxClass != 'gvisor' || - object.spec.versions.all(v, + variables.changedVersions.all(v, has(v.assets) && size(v.assets) > 0 && v.assets.all(arch, 'gvisor' in v.assets[arch])) message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) asset for every architecture of every version under spec.versions[].assets" @@ -42,7 +54,7 @@ spec: # architecture it advertises. - expression: >- object.spec.sandboxClass != 'microvm' || - object.spec.versions.all(v, + variables.changedVersions.all(v, has(v.assets) && size(v.assets) > 0 && v.assets.all(arch, ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image'] diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index 2423d99e03..9075c344d9 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Cluster-wide SandboxConfig for the micro-VM (kata + cloud-hypervisor) sandbox -# class. Unlike sandboxconfig-gvisor.yaml (applied unconditionally by -# ate-setup deploy ate-system), this is opt-in: apply via -# hack/install-microvm-deps.sh --install after staging the asset set -# (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh). A microvm WorkerPool -# resolves it as its class default (or names it via configRef: microvm), so -# a missing/stale config shows on the pool's status. +# Template for the microVM (kata + cloud-hypervisor) SandboxConfig. Unlike +# sandboxconfig-gvisor.yaml.tmpl, which every install applies, this is opt-in: +# hack/install-microvm-deps.sh --install stages the asset set (assemble.sh + +# stage-to-gcs.sh / stage-to-rustfs.sh), then creates +# substrate--microvm from this template, hashed from the rendered +# config, and by default annotates it as the class default. It leaves an +# existing config as it is, so editing this file changes only the configs +# that later builds create. # # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs) # are FETCHED at runtime from the asset bucket ${ASSET_BUCKET_NAME} under @@ -34,13 +35,23 @@ # hack/microvm-assets/assemble.sh; atelet selects the block matching the node's # architecture, and each cluster's bucket holds that arch's binaries at these # paths (staged by hack/install-microvm-deps.sh --install). +# +# To change the sandbox, add a new version and point defaultVersion at it. +# Do not edit or remove the versions already listed. An actor restores its +# memory only on a version with the same assets as the one its snapshot was +# taken on, so dropping that version makes the actor cold boot and lose its +# memory. Remove a version only when it has a known vulnerability, or to make +# room under the limit of 16 versions. One exception remains: the assets live +# at fixed paths, and staging a new asset set overwrites them. So a bump that +# changes the assets has to replace the version until the assets move to +# per-version paths. apiVersion: ate.dev/v1 kind: SandboxConfig metadata: - name: microvm - annotations: - sandboxconfig.ate.dev/is-class-default: "true" + name: ${SANDBOXCONFIG_NAME} + labels: + sandboxconfig.ate.dev/created-by-version: "${SUBSTRATE_VERSION}" spec: sandboxClass: microvm defaultVersion: v2