From c507ea6a41319db70f4abe53d2083c06d5d18697 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Mon, 26 Oct 2026 16:15:36 -0700 Subject: [PATCH 1/9] ate-setup: create a SandboxConfig per release Every upgrade applied gvisor-default in place, so the previous release's config was gone after an upgrade. ate-setup now renders sandboxconfig-gvisor.yaml.tmpl into substrate--gvisor, and --add-default-annotation (default true) adds the class-default annotation. hack/install-microvm-deps.sh does the same for substrate--microvm. atecontroller uses the newest annotated config of a class. Both installers warn when the config gets the annotation but a newer annotated config exists, and when no config of the class has the annotation. They refuse a config that is being deleted. deploy ate-system checks this before it changes the cluster. install-microvm-deps.sh no longer deletes configs. Like the gVisor configs, they go with the SandboxConfig CRD. Signed-off-by: zoezhao --- .../ate/v1/sandboxconfig_validation_test.go | 15 +- benchmarking/automation/README.md | 6 +- benchmarking/automation/orchestrator.py | 23 +-- cmd/ate-setup/internal/cmd/deploy.go | 30 ++- .../demos/countermicrovm/countermicrovm.go | 2 +- .../demos/egressmicrovm/egressmicrovm.go | 2 +- cmd/ate-setup/internal/steps/benchmarks.go | 20 +- cmd/ate-setup/internal/steps/delete.go | 28 +-- cmd/ate-setup/internal/steps/delete_test.go | 89 --------- cmd/ate-setup/internal/steps/deploy.go | 30 +-- cmd/ate-setup/internal/steps/sandboxconfig.go | 184 ++++++++++++++++++ .../internal/steps/sandboxconfig_test.go | 175 +++++++++++++++++ demos/counter/counter-microvm.yaml.tmpl | 7 +- demos/counter/counter.yaml.tmpl | 3 +- demos/egress/egress-microvm.yaml.tmpl | 6 +- hack/install-microvm-deps.sh | 154 +++++++++------ hack/run-microvm-demo.sh | 4 +- internal/e2e/preflight.go | 17 +- ...or.yaml => sandboxconfig-gvisor.yaml.tmpl} | 33 ++-- .../microvm/sandboxconfig-microvm.yaml.tmpl | 32 +-- 20 files changed, 581 insertions(+), 279 deletions(-) create mode 100644 cmd/ate-setup/internal/steps/sandboxconfig.go create mode 100644 cmd/ate-setup/internal/steps/sandboxconfig_test.go rename manifests/ate-install/{sandboxconfig-gvisor.yaml => sandboxconfig-gvisor.yaml.tmpl} (59%) diff --git a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go index 6b6663cec9..1e52ff7ad6 100644 --- a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go +++ b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "testing" "time" @@ -344,14 +345,15 @@ func TestSandboxConfigUpdateRemovingDefaultVersion(t *testing.T) { } } -// TestShippedSandboxConfigManifests guards the SandboxConfigs the install ships -// against the CRD schema and the shipped ValidatingAdmissionPolicy. +// TestShippedSandboxConfigManifests guards the SandboxConfig templates the +// install renders against the CRD schema and the shipped +// ValidatingAdmissionPolicy. func TestShippedSandboxConfigManifests(t *testing.T) { ctx := t.Context() applyVAP(t, ctx) for _, path := range []string{ - "../../../../../manifests/ate-install/sandboxconfig-gvisor.yaml", + "../../../../../manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl", "../../../../../manifests/microvm/sandboxconfig-microvm.yaml.tmpl", } { t.Run(path, func(t *testing.T) { @@ -359,13 +361,14 @@ func TestShippedSandboxConfigManifests(t *testing.T) { if err != nil { t.Fatalf("read manifest: %v", err) } - raw = []byte(strings.NewReplacer("${BUCKET_SCHEME}", "s3", "${BUCKET_NAME}", "test-bucket", "${ASSET_BUCKET_NAME}", "test-bucket").Replace(string(raw))) + raw = []byte(strings.NewReplacer("${SANDBOXCONFIG_NAME}", "test-"+filepath.Base(path), "${BUCKET_SCHEME}", "s3", "${ASSET_BUCKET_NAME}", "test-bucket").Replace(string(raw))) sc := &SandboxConfig{} if err := yaml.UnmarshalStrict(raw, sc); err != nil { t.Fatalf("decode manifest: %v", err) } - if sc.Annotations[SandboxConfigClassDefaultAnnotation] != "true" { - t.Errorf("manifest is not annotated %s=true", SandboxConfigClassDefaultAnnotation) + // The installers add the annotation only with --add-default-annotation. + if _, ok := sc.Annotations[SandboxConfigClassDefaultAnnotation]; ok { + t.Errorf("template sets %s; the installers add it", SandboxConfigClassDefaultAnnotation) } if err := k8sClient.Create(ctx, sc); err != nil { t.Fatalf("Create() unexpected error: %v", err) diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 30a9dfe83c..303a0b9c84 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -32,7 +32,8 @@ To run the cold resume tests from your workstation instead, see Envoy ingress router. For a `microvm` test the orchestrator also runs `hack/install-microvm-deps.sh --install` between the two, which stages kata + cloud-hypervisor + virtiofsd assets to the cluster's object - store bucket and applies the cluster-wide `microvm` SandboxConfig. + store bucket and creates this release's microvm SandboxConfig as the + class default. For a `nighthawk-ingress` test the orchestrator additionally patches the `atenet-router` Deployment right after `deploy_substrate`: `envoyCpu` is the benchmark's independent variable and the shipped manifest sets @@ -49,7 +50,8 @@ To run the cold resume tests from your workstation instead, see variable (a JSON object) if set. A `--metadata` in the test's own `flags` is merged in too, its keys winning on conflict. - Polls until complete/failed/timeout; tails logs; deletes the Job. - - Tears down workloads + microVM deps (if any) + substrate. + - Tears down workloads + substrate. Deleting substrate also removes the + microVM SandboxConfigs with their CRD. - If not the last test, redeploys them so the next run starts clean. ## Choosing a sandbox class diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index c50fdc2946..0dbcfa991d 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -421,19 +421,13 @@ def teardown_substrate() -> None: def install_microvm_deps() -> None: - """Stage kata/cloud-hypervisor assets and apply the cluster-wide - microvm SandboxConfig. Required before a microvm WorkerPool can run - actors; must run after deploy_substrate() (which installs the CRDs).""" + """Stage kata/cloud-hypervisor assets and create this release's microvm + SandboxConfig as the class default. Required before a microvm WorkerPool + can run actors; must run after deploy_substrate() (which installs the + CRDs).""" run(["hack/install-microvm-deps.sh", "--install"]) -def teardown_microvm_deps() -> None: - """Remove the microvm SandboxConfig. Must run before - teardown_substrate(), which deletes the SandboxConfig CRD (and would - prevent this from succeeding via kubectl).""" - run_no_check(["hack/install-microvm-deps.sh", "--delete"]) - - def deploy_workloads( worker_count: int = 1, sandbox_class: str = "gvisor", @@ -632,11 +626,9 @@ def main() -> None: # fire that crashed mid-test (or any other process that left # state behind) would otherwise leak its substrate + workloads # into this run. All teardowns use --ignore-not-found, so - # this is cheap on a clean cluster. Order matters: - # microvm-deps deletes a SandboxConfig CR, which requires the - # SandboxConfig CRD that teardown_substrate removes. + # this is cheap on a clean cluster. teardown_substrate also + # removes the microvm SandboxConfigs with their CRD. teardown_workloads() - teardown_microvm_deps() teardown_substrate() sandbox_class = test.get("sandboxClass", "gvisor") @@ -678,10 +670,7 @@ def main() -> None: finally: # Always tear down, even if deploy or run failed, so the # next test (and the next CronJob fire) starts clean. - # microvm-deps must go before substrate for the same reason - # as above. teardown_workloads() - teardown_microvm_deps() teardown_substrate() duration = time.time() - start_time results.append((test["name"], status, duration, failure_msg)) diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 0d9f38c8eb..dafe99e143 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -84,6 +84,9 @@ Shape the install with the global --atenet-dataplane, --cluster-size, if err != nil { return err } + if opts.AddSandboxConfigDefaultAnnotation, err = cmd.Flags().GetBool(addDefaultAnnotationFlag); err != nil { + return err + } return env.DeployAteSystem(cmd.Context(), opts) }, } @@ -136,12 +139,31 @@ var deployPodCertControllerCmd = &cobra.Command{ }, } +// addDefaultAnnotationFlag names the flag that adds the class-default +// annotation to the release's gVisor SandboxConfig. It is a plain flag rather +// than a recorded setting: each run decides it anew. +const addDefaultAnnotationFlag = "add-default-annotation" + +const addDefaultAnnotationUsage = "Add the sandboxconfig.ate.dev/is-class-default annotation to this release's gVisor SandboxConfig" + var deploySandboxConfigCmd = &cobra.Command{ Use: "sandboxconfig", - Short: "Deploy the SandboxConfig admission policy and the default gVisor SandboxConfig only", - Args: cobra.NoArgs, + Short: "Deploy the SandboxConfig admission policy and this release's gVisor SandboxConfig only", + Long: `Deploy the SandboxConfig admission policy and this release's gVisor +SandboxConfig, substrate--gvisor, rendered from +manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl. + +With --add-default-annotation (the default) the config gets the +sandboxconfig.ate.dev/is-class-default annotation. Every gVisor WorkerPool +without a configRef follows the newest config with the annotation, so a +warning names a newer one if it exists.`, + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - return env.DeploySandboxConfig(cmd.Context()) + addDefaultAnnotation, err := cmd.Flags().GetBool(addDefaultAnnotationFlag) + if err != nil { + return err + } + return env.DeploySandboxConfig(cmd.Context(), addDefaultAnnotation) }, } @@ -181,4 +203,6 @@ func init() { }) config.BindCommandFlags(deployAteSystemPath, deployAteSystemCmd.Flags()) deployAteSystemCmd.Flags().Lookup("setup-csi").NoOptDefVal = "none" + deployAteSystemCmd.Flags().Bool(addDefaultAnnotationFlag, true, addDefaultAnnotationUsage) + deploySandboxConfigCmd.Flags().Bool(addDefaultAnnotationFlag, true, addDefaultAnnotationUsage) } diff --git a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go index 52a5036e62..a6372288b8 100644 --- a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go +++ b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go @@ -14,7 +14,7 @@ // Package countermicrovm installs the micro-VM variant of the counter demo. // -// It needs the cluster-wide `microvm` SandboxConfig that +// It needs the microvm class default SandboxConfig that // hack/install-microvm-deps.sh --install creates. It has no external-volume // option: the CSI path is covered by demo-counter. package countermicrovm diff --git a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go index 7c8146f494..04b37d1744 100644 --- a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go +++ b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go @@ -14,7 +14,7 @@ // Package egressmicrovm installs the micro-VM variant of the egress demo. // -// It needs the cluster-wide `microvm` SandboxConfig that +// It needs the microvm class default SandboxConfig that // hack/install-microvm-deps.sh --install creates. It is a demo of its own // rather than a flag on demo-egress so that it appears in help and in the // `delete all` sweep, and so that both can be installed side by side: the diff --git a/cmd/ate-setup/internal/steps/benchmarks.go b/cmd/ate-setup/internal/steps/benchmarks.go index 68c51d020c..f1acd56f47 100644 --- a/cmd/ate-setup/internal/steps/benchmarks.go +++ b/cmd/ate-setup/internal/steps/benchmarks.go @@ -70,9 +70,9 @@ func (e *Env) DeployBenchmarks(ctx context.Context, opts BenchmarkOptions) error } log.Stepf("deploy_benchmarks (worker_count=%d, sandbox_class=%s)", opts.WorkerCount, opts.SandboxClass) - // The microvm SandboxConfig lives outside the default set installed by - // `deploy ate-system`, which only installs gvisor-default. The workloads - // deploy references it by name and would fail if this were skipped. + // `deploy ate-system` installs only the gVisor SandboxConfig. The + // microvm workload pool follows the microvm class default and would not + // resolve if this were skipped. if opts.SandboxClass == config.SandboxClassMicrovm { if err := e.runScript(ctx, installMicrovmDepScript, "--install"); err != nil { return err @@ -110,17 +110,9 @@ func (e *Env) DeleteBenchmarks(ctx context.Context, opts BenchmarkOptions) error } log.Stepf("delete_benchmarks (sandbox_class=%s)", opts.SandboxClass) - if err := e.runScript(ctx, deployLocustScript, "--delete"); err != nil { - return err - } - // The microvm SandboxConfig is cluster-wide. It is only torn down when - // the caller opted into microvm, and only after the benchmark pool is - // gone. The script leaves it in place if another pool still uses it. - // `delete all` tries again once every demo is gone. - if opts.SandboxClass == config.SandboxClassMicrovm { - return e.runScript(ctx, installMicrovmDepScript, "--delete") - } - return nil + // The microvm SandboxConfigs stay. They are cluster-wide and go with the + // SandboxConfig CRD when the control plane is deleted. + return e.runScript(ctx, deployLocustScript, "--delete") } // runScript executes a repository script from the repository root with the diff --git a/cmd/ate-setup/internal/steps/delete.go b/cmd/ate-setup/internal/steps/delete.go index 7223e8a332..5c43fb528e 100644 --- a/cmd/ate-setup/internal/steps/delete.go +++ b/cmd/ate-setup/internal/steps/delete.go @@ -207,38 +207,14 @@ type Deleter interface { Delete(ctx context.Context, e *Env) error } -// DeleteAll removes every demo, then the microvm SandboxConfig, then the -// control plane. +// DeleteAll removes every demo, then the control plane. func (e *Env) DeleteAll(ctx context.Context, demos []Deleter) error { log.Step("delete_all") - if err := e.deleteDemos(ctx, demos); err != nil { - return err - } - return e.DeleteAteSystem(ctx) -} - -// deleteDemos removes every demo, then tries to delete the microvm -// SandboxConfig. An earlier `delete benchmarks` or `delete demo` may have -// left the config behind because a pool still used it. After the demos, no -// pool does. -// -// The delete is best effort. The script skips a config that is still in use, -// and a failure is only logged, because DeleteAteSystem deletes the config -// with the CRD anyway. -// -// The gvisor config needs no such step. It is installed and deleted with the -// control plane, and DeleteAteSystem strips its finalizer itself. The microvm -// config is managed by hack/install-microvm-deps.sh, whose `kubectl delete` -// would block while a pool still uses it. -func (e *Env) deleteDemos(ctx context.Context, demos []Deleter) error { for _, demo := range demos { if err := demo.Delete(ctx, e); err != nil { return err } } - if err := e.runScript(ctx, installMicrovmDepScript, "--delete"); err != nil { - log.Warnf("while deleting the microvm SandboxConfig: %v", err) - } - return nil + return e.DeleteAteSystem(ctx) } diff --git a/cmd/ate-setup/internal/steps/delete_test.go b/cmd/ate-setup/internal/steps/delete_test.go index 9274cadc6f..a255a23b70 100644 --- a/cmd/ate-setup/internal/steps/delete_test.go +++ b/cmd/ate-setup/internal/steps/delete_test.go @@ -18,8 +18,6 @@ import ( "context" "errors" "fmt" - "os" - "path/filepath" "slices" "strings" "testing" @@ -37,7 +35,6 @@ import ( "k8s.io/client-go/util/retry" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" - "go.ate.dev/substrate/cmd/ate-setup/internal/config" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/internal/clustertrustbundle" ) @@ -193,92 +190,6 @@ func TestReleaseSandboxConfigs(t *testing.T) { } } -// recordingDeleter is a Deleter that appends its name to a log file shared -// with the fake microvm script, so a test can read back the order of calls. -type recordingDeleter struct { - name string - log string - err error -} - -func (d recordingDeleter) Delete(context.Context, *Env) error { - appendLine(d.log, d.name) - return d.err -} - -func appendLine(path, line string) { - f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil { - panic(err) - } - defer f.Close() - fmt.Fprintln(f, line) -} - -func TestDeleteDemos(t *testing.T) { - demoErr := errors.New("demo failed") - tests := []struct { - name string - // demos are the deleters in order. A nil entry's err is nil. - demos []recordingDeleter - // scriptExit is the exit code of the fake install-microvm-deps.sh. - scriptExit int - wantCalls []string - wantErr error - }{ - { - name: "microvm SandboxConfig is deleted after every demo", - demos: []recordingDeleter{{name: "demo-a"}, {name: "demo-b"}}, - wantCalls: []string{"demo-a", "demo-b", "script --delete"}, - }, - { - name: "script failure is logged, not returned", - demos: []recordingDeleter{{name: "demo-a"}}, - scriptExit: 1, - wantCalls: []string{"demo-a", "script --delete"}, - }, - { - name: "a failing demo stops before the script", - demos: []recordingDeleter{{name: "demo-a", err: demoErr}, {name: "demo-b"}}, - wantCalls: []string{"demo-a"}, - wantErr: demoErr, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - root := t.TempDir() - calls := filepath.Join(root, "calls.log") - script := filepath.Join(root, installMicrovmDepScript) - if err := os.MkdirAll(filepath.Dir(script), 0o755); err != nil { - t.Fatal(err) - } - body := fmt.Sprintf("#!/bin/sh\necho \"script $*\" >> %q\nexit %d\n", calls, tt.scriptExit) - if err := os.WriteFile(script, []byte(body), 0o755); err != nil { - t.Fatal(err) - } - demos := make([]Deleter, 0, len(tt.demos)) - for _, d := range tt.demos { - d.log = calls - demos = append(demos, d) - } - e := &Env{Cfg: &config.Config{Root: root}} - - err := e.deleteDemos(t.Context(), demos) - if !errors.Is(err, tt.wantErr) { - t.Fatalf("deleteDemos() error = %v, want %v", err, tt.wantErr) - } - got, _ := os.ReadFile(calls) - gotCalls := strings.Fields(strings.ReplaceAll(strings.TrimSpace(string(got)), "script --delete", "script_--delete")) - for i := range gotCalls { - gotCalls[i] = strings.ReplaceAll(gotCalls[i], "script_--delete", "script --delete") - } - if !slices.Equal(gotCalls, tt.wantCalls) { - t.Errorf("calls = %q, want %q", gotCalls, tt.wantCalls) - } - }) - } -} - // trustBundle returns a bundle for signer the way the API server requires it // to be named: the signer name with "/" replaced by ":", then a suffix. func trustBundle(signer, suffix string) *certsv1.ClusterTrustBundle { diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index bacf30a4e5..fbc45c53a5 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -43,6 +43,9 @@ type DeployOptions struct { // SetupCSI additionally installs the CSI driver (nfs, hostpath, both, none). // Kind only. The hostpath driver is Kind only. SetupCSI string + // AddSandboxConfigDefaultAnnotation adds the class-default annotation to + // the release's gVisor SandboxConfig. See DeploySandboxConfig. + AddSandboxConfigDefaultAnnotation bool } // Validate checks the options that can be checked without configuration or a @@ -86,6 +89,11 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err := e.requireNoWorkerPods(ctx); err != nil { return err } + // DeploySandboxConfig checks the config again partway through. Checking + // here keeps a refusal from leaving a half-upgraded system. + if _, _, err := e.checkSandboxConfig(ctx, opts.AddSandboxConfigDefaultAnnotation); err != nil { + return err + } // The namespace has to exist before RBAC or CRDs are applied. if err := e.EnsureAteSystemNamespace(ctx); err != nil { @@ -112,7 +120,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } - if err := e.DeploySandboxConfig(ctx); err != nil { + if err := e.DeploySandboxConfig(ctx, opts.AddSandboxConfigDefaultAnnotation); err != nil { return err } @@ -581,26 +589,6 @@ func (e *Env) DeployAtenet(ctx context.Context) error { return nil } -// DeploySandboxConfig applies the SandboxConfig admission policy, then the -// default gVisor SandboxConfig. -func (e *Env) DeploySandboxConfig(ctx context.Context) error { - log.Step("deploy_sandboxconfig") - - if err := e.EnsureCRDs(ctx); err != nil { - return err - } - - // Enforce per-class SandboxConfig asset requirements. This is applied - // before any SandboxConfig so the config below is validated too. - if err := e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-validation.yaml")); err != nil { - return err - } - - // Install the cluster-wide default gVisor SandboxConfig that gVisor - // WorkerPools resolve by default unless they pin their own configRef. - return e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-gvisor.yaml")) -} - // EnsureCRDs creates the generated CRDs that are missing. Existing CRDs and // RBAC change only through DeployCRDs, so a component redeploy cannot move them. func (e *Env) EnsureCRDs(ctx context.Context) error { diff --git a/cmd/ate-setup/internal/steps/sandboxconfig.go b/cmd/ate-setup/internal/steps/sandboxconfig.go new file mode 100644 index 0000000000..066f864e50 --- /dev/null +++ b/cmd/ate-setup/internal/steps/sandboxconfig.go @@ -0,0 +1,184 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/dynamic" + + atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/render" +) + +// DeploySandboxConfig applies the SandboxConfig admission policy, then this +// release's gVisor SandboxConfig, substrate--gvisor, rendered from +// sandboxconfig-gvisor.yaml.tmpl. With addDefaultAnnotation it also adds the +// class-default annotation to the config. +func (e *Env) DeploySandboxConfig(ctx context.Context, addDefaultAnnotation bool) error { + log.Step("deploy_sandboxconfig") + + if err := e.EnsureCRDs(ctx); err != nil { + return err + } + + // Enforce per-class SandboxConfig asset requirements. This is applied + // before any SandboxConfig so the config below is validated too. + if err := e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-validation.yaml")); err != nil { + return err + } + + name, warning, err := e.checkSandboxConfig(ctx, addDefaultAnnotation) + if err != nil { + return err + } + + manifest, err := render.Template(e.Cfg.Manifest("sandboxconfig-gvisor.yaml.tmpl"), map[string]string{"SANDBOXCONFIG_NAME": name}, nil) + if err != nil { + return err + } + if err := e.Kube.ApplyBytes(ctx, manifest); err != nil { + return err + } + log.Infof("applied SandboxConfig %s", name) + if addDefaultAnnotation { + // A merge patch rather than part of the apply above. The apply then + // never owns the annotation, so a later run without + // --add-default-annotation leaves it in place. + patch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{"annotations": map[string]any{atev1.SandboxConfigClassDefaultAnnotation: "true"}}, + }) + if err != nil { + return fmt.Errorf("while building the patch for SandboxConfig %s: %w", name, err) + } + if _, err := e.sandboxConfigs().Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{FieldManager: kube.FieldManager}); err != nil { + return fmt.Errorf("while annotating SandboxConfig %s: %w", name, err) + } + log.Infof("annotated SandboxConfig %s %s=true", name, atev1.SandboxConfigClassDefaultAnnotation) + } + if warning != "" { + log.Warnf("%s", warning) + } + return nil +} + +// checkSandboxConfig returns the name of this release's gVisor SandboxConfig +// and checkClassDefault's result for applying it with addDefaultAnnotation. A +// cluster without the SandboxConfig CRD has no configs. +func (e *Env) checkSandboxConfig(ctx context.Context, addDefaultAnnotation bool) (name, warning string, err error) { + class := atev1.SandboxClassGvisor + name, err = sandboxConfigName(e.SubstrateVersion(), class) + if err != nil { + return "", "", err + } + list, err := e.sandboxConfigs().List(ctx, metav1.ListOptions{}) + if apierrors.IsNotFound(err) { + list = &unstructured.UnstructuredList{} + } else if err != nil { + return "", "", fmt.Errorf("while listing SandboxConfigs: %w", err) + } + configs := make([]atev1.SandboxConfig, len(list.Items)) + for i := range list.Items { + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(list.Items[i].Object, &configs[i]); err != nil { + return "", "", fmt.Errorf("while decoding SandboxConfig %s: %w", list.Items[i].GetName(), err) + } + } + warning, err = checkClassDefault(configs, name, class, addDefaultAnnotation) + return name, warning, err +} + +func (e *Env) sandboxConfigs() dynamic.ResourceInterface { + return e.Kube.Dynamic.Resource(atev1.GroupVersion.WithResource("sandboxconfigs")) +} + +// sandboxConfigName returns the name of the SandboxConfig of class that +// Substrate version installs: substrate--, with the version +// lower-cased and every character a name cannot hold replaced by '-'. +func sandboxConfigName(version string, class atev1.SandboxClass) (string, error) { + v := strings.Map(func(r rune) rune { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '-': + return r + case r >= 'A' && r <= 'Z': + return r - 'A' + 'a' + } + return '-' + }, version) + name := fmt.Sprintf("substrate-%s-%s", v, class) + if errs := validation.IsDNS1123Subdomain(name); len(errs) > 0 { + return "", fmt.Errorf("version %q gives the invalid SandboxConfig name %q: %s", version, name, strings.Join(errs, "; ")) + } + return name, nil +} + +// checkClassDefault checks applying the SandboxConfig named name, of class, +// with addDefaultAnnotation. configs are the SandboxConfigs in the cluster. +// +// - It returns an error when a config named name is being deleted. Applying +// it would only update the terminating object. +// - It warns when the config gets the annotation but is not the class +// default afterwards, because a newer default exists. That happens when +// an older release's config is applied again. +// - It warns when the config does not get the annotation and the class has +// no default, so pools of the class without a configRef resolve none. +func checkClassDefault(configs []atev1.SandboxConfig, name string, class atev1.SandboxClass, addDefaultAnnotation bool) (string, error) { + var existing, newest *atev1.SandboxConfig + for i := range configs { + sc := &configs[i] + if sc.Name == name { + existing = sc + } + if sc.Spec.SandboxClass == class && sc.Annotations[atev1.SandboxConfigClassDefaultAnnotation] == "true" && sc.DeletionTimestamp.IsZero() && + (newest == nil || newerSandboxConfig(sc, newest)) { + newest = sc + } + } + if existing != nil && !existing.DeletionTimestamp.IsZero() { + return "", fmt.Errorf("SandboxConfig %s is being deleted; wait until it is gone, then run this again", name) + } + if !addDefaultAnnotation { + if newest == nil { + return fmt.Sprintf("no SandboxConfig of sandbox class %q has the %s annotation, so WorkerPools of the class without a configRef resolve no config", + class, atev1.SandboxConfigClassDefaultAnnotation), nil + } + return "", nil + } + if existing != nil && newest != nil && newest.Name != name && newerSandboxConfig(newest, existing) { + return fmt.Sprintf("SandboxConfig %s has the %s annotation but is not the %s class default, because %s is newer", + name, atev1.SandboxConfigClassDefaultAnnotation, class, newest.Name), nil + } + return "", nil +} + +// newerSandboxConfig reports whether a takes precedence over b as a class +// default: it was created later, or in the same second with a smaller name. +// atecontroller applies the same rule. +func newerSandboxConfig(a, b *atev1.SandboxConfig) bool { + if !a.CreationTimestamp.Equal(&b.CreationTimestamp) { + return b.CreationTimestamp.Before(&a.CreationTimestamp) + } + return a.Name < b.Name +} diff --git a/cmd/ate-setup/internal/steps/sandboxconfig_test.go b/cmd/ate-setup/internal/steps/sandboxconfig_test.go new file mode 100644 index 0000000000..faac0ac799 --- /dev/null +++ b/cmd/ate-setup/internal/steps/sandboxconfig_test.go @@ -0,0 +1,175 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" +) + +func TestSandboxConfigName(t *testing.T) { + tests := []struct { + name string + version string + want string + wantErr bool + }{ + {name: "release tag", version: "v0.4.2", want: "substrate-v0.4.2-gvisor"}, + {name: "git describe of a dirty checkout", version: "v0.4.2-12-gabc1234-dirty", want: "substrate-v0.4.2-12-gabc1234-dirty-gvisor"}, + {name: "upper case and build metadata", version: "V1.0.0+Build", want: "substrate-v1.0.0-build-gvisor"}, + {name: "no git", version: "dev", want: "substrate-dev-gvisor"}, + {name: "dot next to a dash", version: "v1.", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := sandboxConfigName(tt.version, atev1.SandboxClassGvisor) + if (err != nil) != tt.wantErr { + t.Fatalf("sandboxConfigName(%q) error = %v, wantErr %v", tt.version, err, tt.wantErr) + } + if got != tt.want { + t.Errorf("sandboxConfigName(%q) = %q, want %q", tt.version, got, tt.want) + } + }) + } +} + +func TestCheckClassDefault(t *testing.T) { + const name = "substrate-v2-gvisor" + t0 := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + config := func(name string, class atev1.SandboxClass, created time.Time, isDefault, deleting bool) atev1.SandboxConfig { + sc := atev1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: name, CreationTimestamp: metav1.NewTime(created)}, + Spec: atev1.SandboxConfigSpec{SandboxClass: class}, + } + if isDefault { + sc.Annotations = map[string]string{atev1.SandboxConfigClassDefaultAnnotation: "true"} + } + if deleting { + sc.DeletionTimestamp = &metav1.Time{Time: created.Add(time.Hour)} + } + return sc + } + gvisor, microvm := atev1.SandboxClassGvisor, atev1.SandboxClassMicroVM + + tests := []struct { + name string + configs []atev1.SandboxConfig + addDefaultAnnotation bool + // wantWarning is a substring of the warning, empty for none. + wantWarning string + wantErr bool + }{ + { + name: "fresh cluster, annotated", + addDefaultAnnotation: true, + }, + { + name: "fresh cluster, not annotated", + wantWarning: "no SandboxConfig", + }, + { + name: "not annotated with an existing default", + configs: []atev1.SandboxConfig{config("substrate-v1-gvisor", gvisor, t0, true, false)}, + }, + { + name: "not annotated when only another class has a default", + configs: []atev1.SandboxConfig{config("substrate-v1-microvm", microvm, t0, true, false)}, + wantWarning: "no SandboxConfig", + }, + { + name: "not annotated when the only default is being deleted", + configs: []atev1.SandboxConfig{config("substrate-v1-gvisor", gvisor, t0, true, true)}, + wantWarning: "no SandboxConfig", + }, + { + name: "new config annotated over an older default", + configs: []atev1.SandboxConfig{config("substrate-v1-gvisor", gvisor, t0, true, false)}, + addDefaultAnnotation: true, + }, + { + name: "rerun on the config that is already the newest default", + configs: []atev1.SandboxConfig{ + config("substrate-v1-gvisor", gvisor, t0, true, false), + config(name, gvisor, t0.Add(time.Hour), true, false), + }, + addDefaultAnnotation: true, + }, + { + name: "existing config older than the newest of two newer defaults", + configs: []atev1.SandboxConfig{ + config(name, gvisor, t0, false, false), + config("substrate-v3-gvisor", gvisor, t0.Add(time.Hour), true, false), + config("substrate-v4-gvisor", gvisor, t0.Add(2*time.Hour), true, false), + }, + addDefaultAnnotation: true, + wantWarning: "because substrate-v4-gvisor is newer", + }, + { + name: "existing config created with the current default, which has the smaller name", + configs: []atev1.SandboxConfig{ + config(name, gvisor, t0, false, false), + config("substrate-v1-gvisor", gvisor, t0, true, false), + }, + addDefaultAnnotation: true, + wantWarning: "because substrate-v1-gvisor is newer", + }, + { + name: "existing config older than a newer default being deleted", + configs: []atev1.SandboxConfig{ + config(name, gvisor, t0, false, false), + config("substrate-v3-gvisor", gvisor, t0.Add(time.Hour), true, true), + }, + addDefaultAnnotation: true, + }, + { + name: "existing config older than a newer default of another class", + configs: []atev1.SandboxConfig{ + config(name, gvisor, t0, false, false), + config("substrate-v3-microvm", microvm, t0.Add(time.Hour), true, false), + }, + addDefaultAnnotation: true, + }, + { + name: "existing config being deleted, annotated", + configs: []atev1.SandboxConfig{config(name, gvisor, t0, true, true)}, + addDefaultAnnotation: true, + wantErr: true, + }, + { + name: "existing config being deleted, not annotated", + configs: []atev1.SandboxConfig{ + config("substrate-v1-gvisor", gvisor, t0, true, false), + config(name, gvisor, t0.Add(time.Hour), false, true), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + warning, err := checkClassDefault(tt.configs, name, gvisor, tt.addDefaultAnnotation) + if (err != nil) != tt.wantErr { + t.Fatalf("checkClassDefault() error = %v, wantErr %v", err, tt.wantErr) + } + if (warning == "") != (tt.wantWarning == "") || !strings.Contains(warning, tt.wantWarning) { + t.Errorf("checkClassDefault() warning = %q, want it to contain %q", warning, tt.wantWarning) + } + }) + } +} diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index d609850dbf..664bd1b7ee 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -17,9 +17,8 @@ # suspend/resume across pods shows the count continuing — proving the guest # memory snapshot round-tripped. The ActorTemplate is not a CRD: it lives in # counter-microvm-template.yaml.tmpl and is created through the ate API with -# `kubectl ate create actor-template`. The cluster-wide `microvm` -# SandboxConfig referenced by the pool is installed by -# hack/install-microvm-deps.sh +# `kubectl ate create actor-template`. The pool follows the microvm class +# default SandboxConfig, which hack/install-microvm-deps.sh installs. apiVersion: v1 kind: Namespace @@ -41,8 +40,6 @@ spec: replicas: 2 sandboxClasses: - name: microvm - configRef: - name: microvm workerImage: ko://go.ate.dev/substrate/cmd/ateworker-microvm template: # Per-worker resources: these size the worker POD and advertise its diff --git a/demos/counter/counter.yaml.tmpl b/demos/counter/counter.yaml.tmpl index 190ec65e19..cbf77ca037 100644 --- a/demos/counter/counter.yaml.tmpl +++ b/demos/counter/counter.yaml.tmpl @@ -40,7 +40,8 @@ spec: workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor # No configRef: the pool follows the class default, the newest SandboxConfig # of the class annotated sandboxconfig.ate.dev/is-class-default=true - # (gvisor-default as installed), and moves to a new default when one appears. + # (substrate--gvisor as installed), and moves to a new default when + # one appears. sandboxClasses: - name: gvisor template: diff --git a/demos/egress/egress-microvm.yaml.tmpl b/demos/egress/egress-microvm.yaml.tmpl index d4f589b596..66c75b06c4 100644 --- a/demos/egress/egress-microvm.yaml.tmpl +++ b/demos/egress/egress-microvm.yaml.tmpl @@ -16,9 +16,9 @@ # egress demo, so the networking suite's egress tests run against both # sandbox classes. The ActorTemplate is a substrate resource, not a CRD: it # lives in egress-microvm-template.yaml.tmpl and is created through the -# ate API with `kubectl ate create actor-template`. The cluster-wide `microvm` -# SandboxConfig referenced by the pool is installed by -# hack/install-microvm-deps.sh +# ate API with `kubectl ate create actor-template`. The pool follows the +# microvm class default SandboxConfig, which hack/install-microvm-deps.sh +# installs. apiVersion: v1 kind: Namespace diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index 312da2e346..c300a8e466 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -14,26 +14,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Install (or delete) the cluster-wide micro-VM (kata + cloud-hypervisor) -# dependencies. This is opt-in on top of ate-setup deploy ate-system, -# which only installs the default gVisor SandboxConfig. Used by: +# Install the cluster-wide micro-VM (kata + cloud-hypervisor) dependencies. +# This is opt-in on top of ate-setup deploy ate-system, which only installs +# the gVisor SandboxConfig. Used by: # * hack/run-microvm-demo.sh (before applying the counter-microvm demo) # * benchmarking/automation/orchestrator.py (before deploying a microvm # benchmark workload) # # On --install: assembles the asset set (assemble.sh; skipped if OUT already # has them), stages the assets under kata-assets/ to the cluster's object -# store bucket (rustfs on kind, GCS on GKE), and applies the cluster-wide -# `microvm` SandboxConfig referencing those assets. Every asset sha256 is -# pinned in the manifest, so the apply only substitutes the bucket name. +# store bucket (rustfs on kind, GCS on GKE), and creates this release's +# SandboxConfig, substrate--microvm, from the template. The version +# is VERSION, else `git describe`, as for the gVisor config `ate-setup` +# creates. Every asset sha256 is pinned in the template, so the apply only +# substitutes the name and the bucket. With --add-default-annotation (the +# default) the config gets the sandboxconfig.ate.dev/is-class-default +# annotation. # -# On --delete: removes the SandboxConfig from the cluster. The delete is best -# effort. A SandboxConfig that a WorkerPool uses carries a protection -# finalizer, which atecontroller releases only once no pool's status names -# the config. So the script leaves the config in place if any WorkerPool -# still uses it. Otherwise it waits a bounded time for the finalizer to go. -# Bucket contents are left alone (they're inert until a SandboxConfig points -# at them, and re-staging is cheap on next install). +# Nothing here deletes the configs. Like the gVisor configs, they stay until +# the SandboxConfig CRD is deleted with the control plane. # # Like the other hack scripts, this sources .ate-dev-env.sh for the cluster / # registry / bucket settings unless NO_DEV_ENV is set. @@ -49,6 +48,8 @@ # OUT asset dir (default: $PWD/bin/microvm-assets/$ARCH, gitignored). # ATE_INSTALL_KIND "true" for the kind path (stage assets to rustfs); default # false uploads assets to GCS. +# VERSION (optional) Substrate version in the SandboxConfig name +# (default: git describe). set -o errexit -o nounset -o pipefail @@ -67,22 +68,25 @@ ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" usage() { cat <-microvm. + --add-default-annotation With --install, add the class-default annotation + to the config (default: true). + -h, --help Show this message. EOF } action="" +add_default_annotation=true while [[ $# -gt 0 ]]; do case "$1" in --install) action="install" ;; - --delete) action="delete" ;; + --add-default-annotation|--add-default-annotation=true) add_default_annotation=true ;; + --add-default-annotation=false) add_default_annotation=false ;; -h|--help) usage; exit 0 ;; *) echo "Error: unknown argument $1" >&2; usage; exit 1 ;; esac @@ -129,41 +133,68 @@ else ASSET_BUCKET_NAME="${BUCKET_NAME}-assets" fi -if [[ "${action}" == "delete" ]]; then - # A WorkerPool that uses the config holds it. The delete would only mark - # the config Terminating, and `kubectl delete` would wait for it without - # end. A Terminating config also cannot be recreated under its name. So no - # delete is issued at all. The config is left in place and the pools are - # named instead. A cluster without the WorkerPool CRD has no users. - users="$(run_kubectl get workerpools --all-namespaces \ - -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name} {.status.sandboxClasses[*].configRef.name}{"\n"}{end}' 2>/dev/null \ - | awk '{ for (i = 2; i <= NF; i++) if ($i == "microvm") { print $1; break } }' || true)" - if [[ -n "${users}" ]]; then - echo "Warning: the microvm SandboxConfig is left in place; these WorkerPools" >&2 - echo " still use it:" >&2 - while IFS= read -r pool; do echo " ${pool}" >&2; done <<<"${users}" - echo " Delete them, or point their spec.sandboxClasses[].configRef at" >&2 - echo " another microvm SandboxConfig, and run this again." >&2 - exit 0 +DEFAULT_ANNOTATION="sandboxconfig.ate.dev/is-class-default" + +# One line per microvm SandboxConfig: +# name|creationTimestamp|default annotation|deletionTimestamp. +# This fails before the slow assemble and stage if the control plane, and +# with it the SandboxConfig CRD, is not installed. +microvm_configs="$(run_kubectl get sandboxconfigs \ + -o jsonpath='{range .items[?(@.spec.sandboxClass=="microvm")]}{.metadata.name}{"|"}{.metadata.creationTimestamp}{"|"}{.metadata.annotations.sandboxconfig\.ate\.dev/is-class-default}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}')" + +# --- install ---------------------------------------------------------------- + +# Name this release's config the way `ate-setup` names the gVisor one when it +# builds from source: VERSION, else `git describe`, lower-cased, with every +# character a name cannot hold replaced by '-'. Checked here so a bad name +# fails before the stage overwrites the bucket's assets. +version="${VERSION:-$(git describe --tags --always --dirty 2>/dev/null || true)}" +version="$(tr '[:upper:]' '[:lower:]' <<<"${version:-dev}" | sed 's/[^a-z0-9.-]/-/g')" +SANDBOXCONFIG_NAME="substrate-${version}-microvm" +dns_label='[a-z0-9]([-a-z0-9]*[a-z0-9])?' +if [[ ! "${SANDBOXCONFIG_NAME}" =~ ^${dns_label}(\.${dns_label})*$ || ${#SANDBOXCONFIG_NAME} -gt 253 ]]; then + echo "Error: version '${version}' gives the invalid SandboxConfig name ${SANDBOXCONFIG_NAME}." >&2 + exit 1 +fi + +# Check the config before the slow assemble and stage, as +# `ate-setup deploy sandboxconfig` checks the gVisor one: +# * a config of this name must not be being deleted +# * with --add-default-annotation, warn when a newer config has the +# annotation. The newest one is the class default, so this one would not be. +# * without it, warn when no config of the class has the annotation +# Names compare byte by byte, as atecontroller compares them. +export LC_ALL=C +existing_created="" +newest_name="" +newest_created="" +while IFS='|' read -r name created is_default deleting; do + [[ -n "${name}" ]] || continue + if [[ "${name}" == "${SANDBOXCONFIG_NAME}" ]]; then + existing_created="${created}" + if [[ -n "${deleting}" ]]; then + echo "Error: SandboxConfig ${SANDBOXCONFIG_NAME} is being deleted; wait until it" >&2 + echo " is gone, then run this again." >&2 + exit 1 + fi fi - log "Deleting microvm SandboxConfig..." - # Delete by name rather than by manifest so a missing/edited template file - # doesn't block cleanup on an older cluster. atecontroller releases the - # protection finalizer within a reconcile. The timeout turns a missing - # atecontroller into an error instead of a hang. - if ! run_kubectl delete --ignore-not-found --timeout=60s sandboxconfig microvm; then - echo "Error: the microvm SandboxConfig did not finish deleting. It stays" >&2 - echo " Terminating until atecontroller releases its protection" >&2 - echo " finalizer; check that atecontroller is running, then run this" >&2 - echo " again." >&2 - exit 1 + [[ "${is_default}" == "true" && -z "${deleting}" ]] || continue + # RFC 3339 timestamps in UTC sort as strings. + if [[ -z "${newest_name}" || "${created}" > "${newest_created}" ]] || + [[ "${created}" == "${newest_created}" && "${name}" < "${newest_name}" ]]; then + newest_name="${name}" + newest_created="${created}" fi - log "Done. (Bucket assets at ${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/ left in place.)" - exit 0 +done <<<"${microvm_configs}" +warning="" +if [[ "${add_default_annotation}" == "false" && -z "${newest_name}" ]]; then + warning="no microvm SandboxConfig has the ${DEFAULT_ANNOTATION} annotation, so microvm WorkerPools without a configRef resolve no config." +elif [[ "${add_default_annotation}" == "true" && -n "${existing_created}" && -n "${newest_name}" && "${newest_name}" != "${SANDBOXCONFIG_NAME}" ]] && + { [[ "${newest_created}" > "${existing_created}" ]] || + [[ "${newest_created}" == "${existing_created}" && "${newest_name}" < "${SANDBOXCONFIG_NAME}" ]]; }; then + warning="SandboxConfig ${SANDBOXCONFIG_NAME} has the ${DEFAULT_ANNOTATION} annotation but is not the microvm class default, because ${newest_name} is newer." fi -# --- install ---------------------------------------------------------------- - # Target arch: match the images' platform (KO_DEFAULTPLATFORMS is set by # .ate-dev-env.sh on GKE and by the kind wrapper); fall back to the host arch. if [[ -z "${ARCH:-}" ]]; then @@ -215,14 +246,23 @@ else OUT="${OUT}" BUCKET="${ASSET_BUCKET_NAME}" hack/microvm-assets/stage-to-gcs.sh fi -# --- 3. apply the cluster-wide microvm SandboxConfig ----------------------- +# --- 3. create this release's microvm SandboxConfig ------------------------- # Every asset is downloaded rather than built, so all four carry committed, -# reproducible per-arch shas and the bucket's scheme and name are the only -# substitutions left. -log "Applying microvm SandboxConfig from ${MANIFEST_TEMPLATE}..." -sed -e "s|\${BUCKET_SCHEME}|${BUCKET_SCHEME}|g" \ +# reproducible per-arch shas and the name and the bucket's scheme and name are +# the only substitutions left. +log "Applying SandboxConfig ${SANDBOXCONFIG_NAME} from ${MANIFEST_TEMPLATE}..." +sed -e "s|\${SANDBOXCONFIG_NAME}|${SANDBOXCONFIG_NAME}|g" \ + -e "s|\${BUCKET_SCHEME}|${BUCKET_SCHEME}|g" \ -e "s|\${ASSET_BUCKET_NAME}|${ASSET_BUCKET_NAME}|g" \ "${MANIFEST_TEMPLATE}" \ | run_kubectl apply -f - -log "Done. Installed cluster-wide microvm SandboxConfig." +# Annotated outside the template, so a later apply of the template leaves the +# annotation in place. +if [[ "${add_default_annotation}" == "true" ]]; then + run_kubectl annotate --overwrite sandboxconfig "${SANDBOXCONFIG_NAME}" "${DEFAULT_ANNOTATION}=true" +fi +if [[ -n "${warning}" ]]; then + echo "Warning: ${warning}" >&2 +fi +log "Done. Installed SandboxConfig ${SANDBOXCONFIG_NAME}." diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 9ef7334c14..9ba36ad330 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -20,8 +20,8 @@ # # Composes: # 1. ate-setup deploy ate-system (control plane) -# 2. hack/install-microvm-deps.sh --install (asset build/stage + cluster-wide -# microvm SandboxConfig) +# 2. hack/install-microvm-deps.sh --install (asset build/stage + the microvm +# class default SandboxConfig) # 3. Deploy the counter-microvm demo (worker pool manifest, atespace, and # ActorTemplate through the ate API). # diff --git a/internal/e2e/preflight.go b/internal/e2e/preflight.go index 62f7d04e21..22f5c20d8a 100644 --- a/internal/e2e/preflight.go +++ b/internal/e2e/preflight.go @@ -17,11 +17,13 @@ package e2e import ( "context" "fmt" + "slices" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "go.ate.dev/substrate/api/ateapi/v1" + atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" ) // PreflightChecks checks that the test environment is ready for the test suite. @@ -61,15 +63,22 @@ func PreflightChecks() error { return fmt.Errorf("ListActors RPC failed: %v", err) } - // The micro-VM class needs its SandboxConfig registered; without it every + // The microVM class needs a class default SandboxConfig; without it every // fixture in the run is misconfigured. // Note: Whether a node can actually host a micro-VM is left to the scheduler // and any issues are to be reported at runtime. Which device a sandbox class // requires should not be checked here to prevent drift between platforms. if IsMicroVM() { - if _, err := clients.SubstrateK8s.AteV1().SandboxConfigs().Get(ctx, SandboxClassMicroVM, metav1.GetOptions{}); err != nil { - return fmt.Errorf("E2E_SANDBOX_CLASS=%s but SandboxConfig/%s is missing (see docs/dev/microvm-local.md for cluster setup): %w", - SandboxClassMicroVM, SandboxClassMicroVM, err) + list, err := clients.SubstrateK8s.AteV1().SandboxConfigs().List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list SandboxConfigs: %w", err) + } + if !slices.ContainsFunc(list.Items, func(sc atev1.SandboxConfig) bool { + return sc.Spec.SandboxClass == atev1.SandboxClassMicroVM && + sc.Annotations[atev1.SandboxConfigClassDefaultAnnotation] == "true" && sc.DeletionTimestamp.IsZero() + }) { + return fmt.Errorf("E2E_SANDBOX_CLASS=%s but no SandboxConfig of the class is annotated %s=true (see docs/dev/microvm-local.md for cluster setup)", + SandboxClassMicroVM, atev1.SandboxConfigClassDefaultAnnotation) } } diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl similarity index 59% rename from manifests/ate-install/sandboxconfig-gvisor.yaml rename to manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl index a444c303a6..c82075758c 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl @@ -12,26 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Cluster-wide SandboxConfig for the gVisor (runsc) sandbox class, shipped with -# the platform as the class default so gVisor WorkerPools resolve it unless -# they pin a specific configRef. atelet fetches the gVisor release tarball -# (gvisor.tar.zstd: runsc plus the gvisor-bin/ helpers runsc requires next to -# it) matching the worker node's architecture and extracts it locally. To pin a -# different release, add a version below and point defaultVersion at it, or -# create another SandboxConfig and reference it from the WorkerPool. +# Template for the gVisor (runsc) SandboxConfig each Substrate release +# installs. `ate-setup deploy sandboxconfig` and `deploy ate-system` create +# substrate--gvisor from it, and by default annotate it as the class +# default. Editing this file changes only the configs that later releases +# create, and a rerun of the same release updates its own config. # -# A bump adds a version and points defaultVersion at it. Keep the -# previous version unless it has a known vulnerability: an actor restores its -# memory only on the version its snapshot was taken with. +# atelet fetches the gVisor release tarball (gvisor.tar.zstd: runsc plus the +# gvisor-bin/ helpers runsc requires next to it) matching the worker node's +# architecture and extracts it locally. +# +# To change the sandbox, add a new version and point defaultVersion at it. +# Do not edit or remove the versions already listed. An actor restores its +# memory only on a version with the same assets as the one its snapshot was +# taken on, so dropping that version makes the actor cold boot and lose its +# memory. Remove a version only when it has a known vulnerability, or to make +# room under the limit of 16 versions. apiVersion: ate.dev/v1 kind: SandboxConfig metadata: - name: gvisor-default - annotations: - sandboxconfig.ate.dev/is-class-default: "true" + name: ${SANDBOXCONFIG_NAME} spec: sandboxClass: gvisor - defaultVersion: v6 + defaultVersion: gvisor-6 versions: # gVisor release 20261026.0. ateworker-gvisor saves container rootfs overlays # (rootfs/) and durable-dir volumes (fs/) as separate checkpoint bundles. That @@ -39,7 +42,7 @@ spec: # ateworker-gvisor also runs the sandbox with no root container # (google/gvisor#15551). This release's restore picks each container's init # process correctly in such a sandbox (google/gvisor@29b97c66d945). - - name: v6 + - name: gvisor-6 state: Enabled assets: amd64: diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index 2423d99e03..5c61e5afc7 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Cluster-wide SandboxConfig for the micro-VM (kata + cloud-hypervisor) sandbox -# class. Unlike sandboxconfig-gvisor.yaml (applied unconditionally by -# ate-setup deploy ate-system), this is opt-in: apply via -# hack/install-microvm-deps.sh --install after staging the asset set -# (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh). A microvm WorkerPool -# resolves it as its class default (or names it via configRef: microvm), so -# a missing/stale config shows on the pool's status. +# Template for the microVM (kata + cloud-hypervisor) SandboxConfig. Unlike +# sandboxconfig-gvisor.yaml.tmpl, which every install applies, this is opt-in: +# hack/install-microvm-deps.sh --install stages the asset set (assemble.sh + +# stage-to-gcs.sh / stage-to-rustfs.sh), then creates +# substrate--microvm from this template and by default annotates it +# as the class default. Editing this file changes only the configs that later +# releases create. # # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs) # are FETCHED at runtime from the asset bucket ${ASSET_BUCKET_NAME} under @@ -34,18 +34,26 @@ # hack/microvm-assets/assemble.sh; atelet selects the block matching the node's # architecture, and each cluster's bucket holds that arch's binaries at these # paths (staged by hack/install-microvm-deps.sh --install). +# +# To change the sandbox, add a new version and point defaultVersion at it. +# Do not edit or remove the versions already listed. An actor restores its +# memory only on a version with the same assets as the one its snapshot was +# taken on, so dropping that version makes the actor cold boot and lose its +# memory. Remove a version only when it has a known vulnerability, or to make +# room under the limit of 16 versions. One exception remains: the assets live +# at fixed paths, and staging a new asset set overwrites them. So a bump that +# changes the assets has to replace the version until the assets move to +# per-version paths. apiVersion: ate.dev/v1 kind: SandboxConfig metadata: - name: microvm - annotations: - sandboxconfig.ate.dev/is-class-default: "true" + name: ${SANDBOXCONFIG_NAME} spec: sandboxClass: microvm - defaultVersion: v2 + defaultVersion: microvm-2 versions: - - name: v2 + - name: microvm-2 state: Enabled assets: arm64: From 10608d3b51a8871a8dd8748088a4b6951cf043bc Mon Sep 17 00:00:00 2001 From: zoezhao Date: Mon, 26 Oct 2026 15:51:35 -0700 Subject: [PATCH 2/9] docs: describe per-release SandboxConfigs Update the upgrade, rollback and version skew docs for the SandboxConfig each release creates. A rollback creates a new config from the old release's template and makes it the class default. Signed-off-by: zoezhao --- docs/api-guide.md | 17 +++++++++-------- docs/dev/microvm-local.md | 2 +- docs/upgrade.md | 16 ++++++++++++---- docs/version-skew-policy.md | 6 +++--- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/docs/api-guide.md b/docs/api-guide.md index 7fba29d68a..f2b68015d6 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -61,8 +61,9 @@ spec: workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor - configRef: - name: gvisor-default + # Optional. Pins the pool to one SandboxConfig instead of the class default. + # configRef: + # name: substrate-v0.4.2-gvisor template: labels: project: agent-platform @@ -522,7 +523,7 @@ Each entry of `versions` (`SandboxVersionConfig`): | `state` | `string` | Optional. `Enabled` (default) or `Disabled`. The version named by `defaultVersion` cannot be `Disabled`. | | `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A microVM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, is unreachable, or fails with an error it does not recognize, such as GCS refusing its credentials for a public bucket) is then read anonymously, which serves public buckets such as `gs://gvisor`; any other refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (a snapshot location followed by `/atespaces/`), so assets must not be stored there. On GKE, setup-gcp lets the plugin read only the asset bucket, `-assets`, so keep assets there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | -A cluster-wide gVisor `SandboxConfig` (`gvisor-default`, annotated as the class default) is installed with the platform, so gVisor `WorkerPool`s use it automatically without any extra setup. +Each Substrate release installs its own gVisor `SandboxConfig`, `substrate--gvisor`, rendered from [`manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl`](../manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl). `ate-setup deploy ate-system` and `ate-setup deploy sandboxconfig` add the class-default annotation to it unless run with `--add-default-annotation=false`, so gVisor `WorkerPool`s use it without any extra setup. If a newer config of the class already has the annotation, the newer one stays the default, and `ate-setup` warns. Configs from earlier releases stay in the cluster. ### Class default @@ -544,7 +545,7 @@ atecontroller keeps a `SandboxConfig` from being deleted while a `WorkerPool` us A delete cannot be undone, and an object that is being deleted cannot be recreated under the same name. Applying its manifest again only updates the terminating object. To recover from deleting a class default that pools still use: 1. Create a replacement under a different name with the `is-class-default` annotation. Pools move to it on their own, the finalizer is released, and the old config disappears. -2. Once the old config is gone, the next `ate-setup` upgrade recreates the shipped config under its original name. Pools move back to it, because it is then the newest default. +2. Once the old config is gone, `ate-setup deploy sandboxconfig` recreates the release's config under its original name. Pools move back to it, because it is then the newest default. atecontroller alone adds and releases the finalizer. While it is down, `kubectl delete sandboxconfig` of a config that carries the finalizer hangs until atecontroller is back. `ate-setup` strips the finalizer from every config when it deletes the control plane. @@ -554,14 +555,14 @@ atecontroller alone adds and releases the finalizer. While it is down, `kubectl apiVersion: ate.dev/v1 kind: SandboxConfig metadata: - name: gvisor-default + name: substrate-v0.4.2-gvisor annotations: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - defaultVersion: v6 + defaultVersion: gvisor-6 versions: - - name: v6 + - name: gvisor-6 state: Enabled assets: amd64: @@ -578,7 +579,7 @@ spec: A `microvm` `SandboxConfig` supplies the [Kata Containers](https://katacontainers.io/) + [Cloud Hypervisor](https://www.cloudhypervisor.org/) toolchain instead of `runsc`. Each architecture of each version must define the full asset set — `cloud-hypervisor`, `virtiofsd`, `kata-kernel`, and `kata-image` — which a `ValidatingAdmissionPolicy` enforces at apply time. Worker pods for a microVM pool require `/dev/kvm` and nested-virtualization-capable nodes. The controller requests those devices on the pod automatically, and atelet advertises them only where they exist, so placement follows the hardware rather than a node label. Clusters that reserve nested-virt nodes with an `ate.dev/sandboxClass=microvm` taint are still tolerated: advertising a device attracts these pods to capable nodes but repels nothing else from them. The same convention applies to every class: worker pods of a pool tolerate `ate.dev/sandboxClass=:NoSchedule`, so a cluster can reserve a node pool per sandbox class with that taint, and the atelet DaemonSet tolerates the key for any value. On GKE, a pool can also select a ComputeClass with `template.nodeSelector` (`cloud.google.com/compute-class: `): GKE taints the nodes it creates for the class `cloud.google.com/compute-class=:NoSchedule` and adds the matching toleration to the worker pods, and the atelet DaemonSet tolerates that key as well. -See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. +`hack/install-microvm-deps.sh --install` stages the assets and creates `substrate--microvm` from [`manifests/microvm/sandboxconfig-microvm.yaml.tmpl`](../manifests/microvm/sandboxconfig-microvm.yaml.tmpl), with the class-default annotation unless run with `--add-default-annotation=false`. See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. --- diff --git a/docs/dev/microvm-local.md b/docs/dev/microvm-local.md index 66aba29851..08b897ab98 100644 --- a/docs/dev/microvm-local.md +++ b/docs/dev/microvm-local.md @@ -60,7 +60,7 @@ This is a one-shot bring-up: it deploys the control plane, installs the cluster-wide microVM deps via `hack/install-microvm-deps.sh` — assembling the guest runtime assets for your architecture (skipped if already present under `bin/microvm-assets/`), staging them into the in-cluster rustfs bucket, and -applying the `microvm` `SandboxConfig` — then deploys the demo worker pool + +creating the microvm class default `SandboxConfig` — then deploys the demo worker pool + template. ### 4. Verify diff --git a/docs/upgrade.md b/docs/upgrade.md index f440bc5540..928760b3af 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -218,11 +218,15 @@ kubectl -n ate-system rollout status deploy/atenet-egress go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml ``` -This applies the SandboxConfig admission policy and the default SandboxConfig. It does not roll workers. If the release changes a sandbox version, new actors start on the new version, and actors suspended on the previous one still restore on it. +This applies the SandboxConfig admission policy and creates this release's gVisor SandboxConfig, `substrate-$NEW_RELEASE-gvisor`, as the gVisor class default. The old release's config stays in the cluster. gVisor pools without a `configRef` move to the new config without rolling their workers. New actors start on its default version, and actors suspended on a version it still lists restore on it. Pools pinned to a config with `configRef` stay on it until you change their `configRef`. + +To create the config without moving any pool, add `--add-default-annotation=false`. Later, make it the default with `kubectl annotate sandboxconfig substrate-$NEW_RELEASE-gvisor sandboxconfig.ate.dev/is-class-default=true`. + +If the cluster runs microVM pools, also run `hack/install-microvm-deps.sh --install` from the new checkout. It creates `substrate-$NEW_RELEASE-microvm` the same way and takes the same `--add-default-annotation` flag. It first stages the assets at fixed paths in the bucket. If the release changes them, earlier microVM configs no longer match the bucket, and atelet cannot fetch their assets on nodes that have not cached them. MicroVM pools pinned to an earlier config, or left on one by `--add-default-annotation=false`, then need to move to the new config. - **Done:** the command exits 0. - **Stuck:** the command fails. [Roll back step 4](#roll-back-step-4). -- **Actors:** a suspended actor whose sandbox version the release drops resumes without its memory, keeping its files. +- **Actors:** a suspended actor whose sandbox version the new config does not list resumes without its memory, keeping its files. #### 4e. The rest of the system bundle @@ -268,14 +272,18 @@ cd ~/ate-upgrade/old Tell actor owners to stop using API fields new in this release, because the old API server rejects them. Then: ```bash -go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/old.yaml +ROLLBACK=substrate-$NEW_RELEASE-gvisor-rollback +sed "s/\${SANDBOXCONFIG_NAME}/$ROLLBACK/" manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl | kubectl create -f - +kubectl annotate sandboxconfig $ROLLBACK sandboxconfig.ate.dev/is-class-default=true go run ./cmd/ate-setup deploy atenet --config ~/ate-upgrade/old.yaml go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/old.yaml go run ./cmd/ate-setup deploy apiserver --config ~/ate-upgrade/old.yaml go install ./cmd/kubectl-ate ``` -`deploy sandboxconfig` removes a sandbox version new in this release, so actors suspended on it resume without their memory. `deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. No pool rolls. +The first two `kubectl` commands create a new gVisor SandboxConfig from the old release's template and make it the class default. It is the newest default, so gVisor pools without a `configRef` move to it. Running `deploy sandboxconfig` from the old checkout does not do this. The old release's config already exists and is older than the new one, so it stays behind the new default even with the annotation, and `deploy sandboxconfig` warns about it. Actors suspended on a version the rollback config does not list resume without their memory. To keep restoring snapshots taken on a version new in this release, copy that version's entry from the new checkout's template into the rollback config before you create it. Pools pinned to `substrate-$NEW_RELEASE-gvisor` stay on it until you change their `configRef`. Roll back microVM pools the same way, from `manifests/microvm/sandboxconfig-microvm.yaml.tmpl`, with `${BUCKET_SCHEME}` and `${ASSET_BUCKET_NAME}` filled in as `hack/install-microvm-deps.sh` fills them. If the release changed the microVM assets, first stage the old ones from the old checkout with `hack/microvm-assets/assemble.sh` and the stage script, as `hack/install-microvm-deps.sh --install` runs them. + +`deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. No pool rolls. Fields that the new release added to WorkerPools, SandboxConfigs or CSIDriverConfigs are dropped from those objects. Remove them from your YAML files too, or `kubectl apply` rejects the files. diff --git a/docs/version-skew-policy.md b/docs/version-skew-policy.md index 64427c1b7a..5dccd16aa8 100644 --- a/docs/version-skew-policy.md +++ b/docs/version-skew-policy.md @@ -78,11 +78,11 @@ Between compatible releases, a CRD change only adds optional fields. The CRDs se ### SandboxConfig -Each sandbox class has a default SandboxConfig, such as `gvisor-default`, that lists the sandbox versions a cluster may run. A release that changes a sandbox version adds the new one, points `defaultVersion` at it, and keeps the previous release's version unless that version has a known vulnerability. +Each release creates its own SandboxConfig per sandbox class, such as `substrate-v0.4.2-gvisor`, from a template in the repository. The config lists the sandbox versions a cluster may run. By default the upgrade annotates it as the class default, and pools without a `configRef` move to it. The previous release's config stays in the cluster. A release that changes a sandbox version adds the new one to the template, points `defaultVersion` at it, and keeps the previous release's version unless that version has a known vulnerability. -A kept version's assets must stay fetchable, so a release does not overwrite them. The upgrade applies these lists. That does not change the pod template, so workers do not roll. +A kept version's assets must stay fetchable, so a release does not overwrite them. The microVM assets live at fixed paths in the cluster's bucket, so a microVM release that changes them replaces the version instead. Moving a pool to a new config does not change the pod template, so workers do not roll. -An actor that starts without memory state, new or cold booting, gets the default version. An actor restoring memory state gets the version its snapshot was taken with, and stays on it until it next starts without memory state. If that version is gone, it cold boots, keeping its files and losing its memory. Rollback applies the N lists again, which removes the N+1 versions. +An actor that starts without memory state, new or cold booting, gets the default version. An actor restoring memory state gets a version with the same assets as the one its snapshot was taken on, and stays on it until it next starts without memory state. If the pool's config has no such version, the actor cold boots, keeping its files and losing its memory. Rollback creates a new config from the N template and makes it the class default, which removes the N+1 versions unless the rollback config keeps them. From d416ed4344363760e87aac159f20e8caf6e4b0da Mon Sep 17 00:00:00 2001 From: zoezhao Date: Mon, 26 Oct 2026 16:44:29 -0700 Subject: [PATCH 3/9] ate-setup: check the class default after applying the SandboxConfig The class-default warning now reads the cluster after the apply, so it sees the config's real creation time and annotations. ate-setup passes VERSION to the scripts it runs, so the microVM config gets the same version as the gVisor one with prebuilt images. The upgrade runbook passes --add-default-annotation=false to step 4e as well, gives install-microvm-deps.sh the bucket and version, makes the rollback commands safe to rerun, and covers upgrading again after a rollback. Prose spells microVM. Signed-off-by: zoezhao --- benchmarking/automation/README.md | 2 +- benchmarking/automation/orchestrator.py | 6 +- cmd/ate-setup/commands.md | 5 + .../demos/countermicrovm/countermicrovm.go | 2 +- .../demos/egressmicrovm/egressmicrovm.go | 2 +- cmd/ate-setup/internal/steps/benchmarks.go | 8 +- cmd/ate-setup/internal/steps/deploy.go | 2 +- cmd/ate-setup/internal/steps/sandboxconfig.go | 77 +++++++++------- .../internal/steps/sandboxconfig_test.go | 85 +++++++++-------- demos/counter/counter-microvm.yaml.tmpl | 2 +- demos/egress/egress-microvm.yaml.tmpl | 2 +- docs/dev/microvm-local.md | 2 +- docs/upgrade.md | 22 +++-- hack/install-microvm-deps.sh | 91 ++++++++++--------- hack/run-microvm-demo.sh | 2 +- 15 files changed, 180 insertions(+), 130 deletions(-) diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 303a0b9c84..c0ee3edc7b 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -32,7 +32,7 @@ To run the cold resume tests from your workstation instead, see Envoy ingress router. For a `microvm` test the orchestrator also runs `hack/install-microvm-deps.sh --install` between the two, which stages kata + cloud-hypervisor + virtiofsd assets to the cluster's object - store bucket and creates this release's microvm SandboxConfig as the + store bucket and creates this release's microVM SandboxConfig as the class default. For a `nighthawk-ingress` test the orchestrator additionally patches the `atenet-router` Deployment right after `deploy_substrate`: `envoyCpu` diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index 0dbcfa991d..6f5234346f 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -421,8 +421,8 @@ def teardown_substrate() -> None: def install_microvm_deps() -> None: - """Stage kata/cloud-hypervisor assets and create this release's microvm - SandboxConfig as the class default. Required before a microvm WorkerPool + """Stage kata/cloud-hypervisor assets and create this release's microVM + SandboxConfig as the class default. Required before a microVM WorkerPool can run actors; must run after deploy_substrate() (which installs the CRDs).""" run(["hack/install-microvm-deps.sh", "--install"]) @@ -627,7 +627,7 @@ def main() -> None: # state behind) would otherwise leak its substrate + workloads # into this run. All teardowns use --ignore-not-found, so # this is cheap on a clean cluster. teardown_substrate also - # removes the microvm SandboxConfigs with their CRD. + # removes the microVM SandboxConfigs with their CRD. teardown_workloads() teardown_substrate() diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 87672d5217..c685a3bc1c 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -113,6 +113,11 @@ apiserver, the controller, atenet, and atelet. It creates every `create` resource below on the way, so those subcommands are only needed to redo one on a running cluster. +`deploy ate-system` and `deploy sandboxconfig` create this release's gVisor +SandboxConfig, `substrate--gvisor`, with the +`sandboxconfig.ate.dev/is-class-default` annotation. Pass +`--add-default-annotation=false` to leave the annotation off. + ## Publish - `publish worker-images` diff --git a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go index a6372288b8..16fe9fa899 100644 --- a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go +++ b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go @@ -14,7 +14,7 @@ // Package countermicrovm installs the micro-VM variant of the counter demo. // -// It needs the microvm class default SandboxConfig that +// It needs the microVM class default SandboxConfig that // hack/install-microvm-deps.sh --install creates. It has no external-volume // option: the CSI path is covered by demo-counter. package countermicrovm diff --git a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go index 04b37d1744..4893e1a346 100644 --- a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go +++ b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go @@ -14,7 +14,7 @@ // Package egressmicrovm installs the micro-VM variant of the egress demo. // -// It needs the microvm class default SandboxConfig that +// It needs the microVM class default SandboxConfig that // hack/install-microvm-deps.sh --install creates. It is a demo of its own // rather than a flag on demo-egress so that it appears in help and in the // `delete all` sweep, and so that both can be installed side by side: the diff --git a/cmd/ate-setup/internal/steps/benchmarks.go b/cmd/ate-setup/internal/steps/benchmarks.go index f1acd56f47..3388c22c90 100644 --- a/cmd/ate-setup/internal/steps/benchmarks.go +++ b/cmd/ate-setup/internal/steps/benchmarks.go @@ -71,7 +71,7 @@ func (e *Env) DeployBenchmarks(ctx context.Context, opts BenchmarkOptions) error log.Stepf("deploy_benchmarks (worker_count=%d, sandbox_class=%s)", opts.WorkerCount, opts.SandboxClass) // `deploy ate-system` installs only the gVisor SandboxConfig. The - // microvm workload pool follows the microvm class default and would not + // microVM workload pool follows the microVM class default and would not // resolve if this were skipped. if opts.SandboxClass == config.SandboxClassMicrovm { if err := e.runScript(ctx, installMicrovmDepScript, "--install"); err != nil { @@ -110,7 +110,7 @@ func (e *Env) DeleteBenchmarks(ctx context.Context, opts BenchmarkOptions) error } log.Stepf("delete_benchmarks (sandbox_class=%s)", opts.SandboxClass) - // The microvm SandboxConfigs stay. They are cluster-wide and go with the + // The microVM SandboxConfigs stay. They are cluster-wide and go with the // SandboxConfig CRD when the control plane is deleted. return e.runScript(ctx, deployLocustScript, "--delete") } @@ -120,7 +120,9 @@ func (e *Env) DeleteBenchmarks(ctx context.Context, opts BenchmarkOptions) error func (e *Env) runScript(ctx context.Context, relPath string, args ...string) error { cmd := exec.CommandContext(ctx, e.Cfg.Path(relPath), args...) cmd.Dir = e.Cfg.Root - cmd.Env = e.Cfg.ScriptEnv() + // install-microvm-deps.sh names its SandboxConfig after VERSION. Set it + // so the name matches the gVisor config's, also with prebuilt images. + cmd.Env = append(e.Cfg.ScriptEnv(), "VERSION="+e.SubstrateVersion()) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index fbc45c53a5..c9b302bddc 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -91,7 +91,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { } // DeploySandboxConfig checks the config again partway through. Checking // here keeps a refusal from leaving a half-upgraded system. - if _, _, err := e.checkSandboxConfig(ctx, opts.AddSandboxConfigDefaultAnnotation); err != nil { + if _, err := e.checkSandboxConfig(ctx); err != nil { return err } diff --git a/cmd/ate-setup/internal/steps/sandboxconfig.go b/cmd/ate-setup/internal/steps/sandboxconfig.go index 066f864e50..82ea34118f 100644 --- a/cmd/ate-setup/internal/steps/sandboxconfig.go +++ b/cmd/ate-setup/internal/steps/sandboxconfig.go @@ -22,7 +22,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" @@ -51,7 +50,7 @@ func (e *Env) DeploySandboxConfig(ctx context.Context, addDefaultAnnotation bool return err } - name, warning, err := e.checkSandboxConfig(ctx, addDefaultAnnotation) + name, err := e.checkSandboxConfig(ctx) if err != nil { return err } @@ -79,35 +78,54 @@ func (e *Env) DeploySandboxConfig(ctx context.Context, addDefaultAnnotation bool } log.Infof("annotated SandboxConfig %s %s=true", name, atev1.SandboxConfigClassDefaultAnnotation) } + // Checked after the apply, against the config's real creation time and + // annotations. + configs, err := e.listSandboxConfigs(ctx) + if err != nil { + return err + } + warning, err := checkClassDefault(configs, name, atev1.SandboxClassGvisor) if warning != "" { log.Warnf("%s", warning) } - return nil + return err } -// checkSandboxConfig returns the name of this release's gVisor SandboxConfig -// and checkClassDefault's result for applying it with addDefaultAnnotation. A -// cluster without the SandboxConfig CRD has no configs. -func (e *Env) checkSandboxConfig(ctx context.Context, addDefaultAnnotation bool) (name, warning string, err error) { +// checkSandboxConfig returns the name of this release's gVisor SandboxConfig. +// It fails when checkClassDefault does. +func (e *Env) checkSandboxConfig(ctx context.Context) (string, error) { class := atev1.SandboxClassGvisor - name, err = sandboxConfigName(e.SubstrateVersion(), class) + name, err := sandboxConfigName(e.SubstrateVersion(), class) + if err != nil { + return "", err + } + configs, err := e.listSandboxConfigs(ctx) if err != nil { - return "", "", err + return "", err } + // The warning is about the cluster before the apply. + if _, err := checkClassDefault(configs, name, class); err != nil { + return "", err + } + return name, nil +} + +// listSandboxConfigs returns the SandboxConfigs in the cluster. A cluster +// without the SandboxConfig CRD has none. +func (e *Env) listSandboxConfigs(ctx context.Context) ([]atev1.SandboxConfig, error) { list, err := e.sandboxConfigs().List(ctx, metav1.ListOptions{}) if apierrors.IsNotFound(err) { - list = &unstructured.UnstructuredList{} + return nil, nil } else if err != nil { - return "", "", fmt.Errorf("while listing SandboxConfigs: %w", err) + return nil, fmt.Errorf("while listing SandboxConfigs: %w", err) } configs := make([]atev1.SandboxConfig, len(list.Items)) for i := range list.Items { if err := runtime.DefaultUnstructuredConverter.FromUnstructured(list.Items[i].Object, &configs[i]); err != nil { - return "", "", fmt.Errorf("while decoding SandboxConfig %s: %w", list.Items[i].GetName(), err) + return nil, fmt.Errorf("while decoding SandboxConfig %s: %w", list.Items[i].GetName(), err) } } - warning, err = checkClassDefault(configs, name, class, addDefaultAnnotation) - return name, warning, err + return configs, nil } func (e *Env) sandboxConfigs() dynamic.ResourceInterface { @@ -134,17 +152,17 @@ func sandboxConfigName(version string, class atev1.SandboxClass) (string, error) return name, nil } -// checkClassDefault checks applying the SandboxConfig named name, of class, -// with addDefaultAnnotation. configs are the SandboxConfigs in the cluster. +// checkClassDefault checks the SandboxConfig named name, of class. configs +// are the SandboxConfigs in the cluster. // -// - It returns an error when a config named name is being deleted. Applying -// it would only update the terminating object. -// - It warns when the config gets the annotation but is not the class -// default afterwards, because a newer default exists. That happens when -// an older release's config is applied again. -// - It warns when the config does not get the annotation and the class has -// no default, so pools of the class without a configRef resolve none. -func checkClassDefault(configs []atev1.SandboxConfig, name string, class atev1.SandboxClass, addDefaultAnnotation bool) (string, error) { +// - It returns an error when the config is being deleted. Applying it would +// only update the terminating object. +// - It warns when the config has the class-default annotation but is not +// the class default, because a newer default exists. That happens when an +// older release's config is applied again. +// - It warns when the class has no default, so pools of the class without a +// configRef resolve none. +func checkClassDefault(configs []atev1.SandboxConfig, name string, class atev1.SandboxClass) (string, error) { var existing, newest *atev1.SandboxConfig for i := range configs { sc := &configs[i] @@ -159,14 +177,11 @@ func checkClassDefault(configs []atev1.SandboxConfig, name string, class atev1.S if existing != nil && !existing.DeletionTimestamp.IsZero() { return "", fmt.Errorf("SandboxConfig %s is being deleted; wait until it is gone, then run this again", name) } - if !addDefaultAnnotation { - if newest == nil { - return fmt.Sprintf("no SandboxConfig of sandbox class %q has the %s annotation, so WorkerPools of the class without a configRef resolve no config", - class, atev1.SandboxConfigClassDefaultAnnotation), nil - } - return "", nil + if newest == nil { + return fmt.Sprintf("no SandboxConfig of sandbox class %q has the %s annotation, so WorkerPools of the class without a configRef resolve no config", + class, atev1.SandboxConfigClassDefaultAnnotation), nil } - if existing != nil && newest != nil && newest.Name != name && newerSandboxConfig(newest, existing) { + if existing != nil && existing.Annotations[atev1.SandboxConfigClassDefaultAnnotation] == "true" && newest.Name != name { return fmt.Sprintf("SandboxConfig %s has the %s annotation but is not the %s class default, because %s is newer", name, atev1.SandboxConfigClassDefaultAnnotation, class, newest.Name), nil } diff --git a/cmd/ate-setup/internal/steps/sandboxconfig_test.go b/cmd/ate-setup/internal/steps/sandboxconfig_test.go index faac0ac799..e019b9a8b5 100644 --- a/cmd/ate-setup/internal/steps/sandboxconfig_test.go +++ b/cmd/ate-setup/internal/steps/sandboxconfig_test.go @@ -69,91 +69,100 @@ func TestCheckClassDefault(t *testing.T) { gvisor, microvm := atev1.SandboxClassGvisor, atev1.SandboxClassMicroVM tests := []struct { - name string - configs []atev1.SandboxConfig - addDefaultAnnotation bool + name string + configs []atev1.SandboxConfig // wantWarning is a substring of the warning, empty for none. wantWarning string wantErr bool }{ { - name: "fresh cluster, annotated", - addDefaultAnnotation: true, - }, - { - name: "fresh cluster, not annotated", + name: "fresh cluster", wantWarning: "no SandboxConfig", }, { - name: "not annotated with an existing default", - configs: []atev1.SandboxConfig{config("substrate-v1-gvisor", gvisor, t0, true, false)}, + name: "the only config, annotated", + configs: []atev1.SandboxConfig{config(name, gvisor, t0, true, false)}, }, { - name: "not annotated when only another class has a default", - configs: []atev1.SandboxConfig{config("substrate-v1-microvm", microvm, t0, true, false)}, + name: "the only config, not annotated", + configs: []atev1.SandboxConfig{config(name, gvisor, t0, false, false)}, wantWarning: "no SandboxConfig", }, { - name: "not annotated when the only default is being deleted", - configs: []atev1.SandboxConfig{config("substrate-v1-gvisor", gvisor, t0, true, true)}, + name: "not annotated with an existing default", + configs: []atev1.SandboxConfig{ + config("substrate-v1-gvisor", gvisor, t0, true, false), + config(name, gvisor, t0.Add(time.Hour), false, false), + }, + }, + { + name: "not annotated when only another class has a default", + configs: []atev1.SandboxConfig{ + config("substrate-v1-microvm", microvm, t0, true, false), + config(name, gvisor, t0.Add(time.Hour), false, false), + }, wantWarning: "no SandboxConfig", }, { - name: "new config annotated over an older default", - configs: []atev1.SandboxConfig{config("substrate-v1-gvisor", gvisor, t0, true, false)}, - addDefaultAnnotation: true, + name: "not annotated when the only default is being deleted", + configs: []atev1.SandboxConfig{ + config("substrate-v1-gvisor", gvisor, t0, true, true), + config(name, gvisor, t0.Add(time.Hour), false, false), + }, + wantWarning: "no SandboxConfig", }, { - name: "rerun on the config that is already the newest default", + name: "annotated over an older default", configs: []atev1.SandboxConfig{ config("substrate-v1-gvisor", gvisor, t0, true, false), config(name, gvisor, t0.Add(time.Hour), true, false), }, - addDefaultAnnotation: true, }, { - name: "existing config older than the newest of two newer defaults", + name: "annotated, older than the newest of two newer defaults", configs: []atev1.SandboxConfig{ - config(name, gvisor, t0, false, false), + config(name, gvisor, t0, true, false), config("substrate-v3-gvisor", gvisor, t0.Add(time.Hour), true, false), config("substrate-v4-gvisor", gvisor, t0.Add(2*time.Hour), true, false), }, - addDefaultAnnotation: true, - wantWarning: "because substrate-v4-gvisor is newer", + wantWarning: "because substrate-v4-gvisor is newer", }, { - name: "existing config created with the current default, which has the smaller name", + name: "annotated, created in the same second as a default with a smaller name", configs: []atev1.SandboxConfig{ - config(name, gvisor, t0, false, false), config("substrate-v1-gvisor", gvisor, t0, true, false), + config(name, gvisor, t0, true, false), }, - addDefaultAnnotation: true, - wantWarning: "because substrate-v1-gvisor is newer", + wantWarning: "because substrate-v1-gvisor is newer", }, { - name: "existing config older than a newer default being deleted", + name: "not annotated, older than a newer default", configs: []atev1.SandboxConfig{ config(name, gvisor, t0, false, false), + config("substrate-v3-gvisor", gvisor, t0.Add(time.Hour), true, false), + }, + }, + { + name: "annotated, older than a newer default being deleted", + configs: []atev1.SandboxConfig{ + config(name, gvisor, t0, true, false), config("substrate-v3-gvisor", gvisor, t0.Add(time.Hour), true, true), }, - addDefaultAnnotation: true, }, { - name: "existing config older than a newer default of another class", + name: "annotated, older than a newer default of another class", configs: []atev1.SandboxConfig{ - config(name, gvisor, t0, false, false), + config(name, gvisor, t0, true, false), config("substrate-v3-microvm", microvm, t0.Add(time.Hour), true, false), }, - addDefaultAnnotation: true, }, { - name: "existing config being deleted, annotated", - configs: []atev1.SandboxConfig{config(name, gvisor, t0, true, true)}, - addDefaultAnnotation: true, - wantErr: true, + name: "being deleted, annotated", + configs: []atev1.SandboxConfig{config(name, gvisor, t0, true, true)}, + wantErr: true, }, { - name: "existing config being deleted, not annotated", + name: "being deleted, not annotated", configs: []atev1.SandboxConfig{ config("substrate-v1-gvisor", gvisor, t0, true, false), config(name, gvisor, t0.Add(time.Hour), false, true), @@ -163,7 +172,7 @@ func TestCheckClassDefault(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - warning, err := checkClassDefault(tt.configs, name, gvisor, tt.addDefaultAnnotation) + warning, err := checkClassDefault(tt.configs, name, gvisor) if (err != nil) != tt.wantErr { t.Fatalf("checkClassDefault() error = %v, wantErr %v", err, tt.wantErr) } diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index 664bd1b7ee..78e109f177 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -17,7 +17,7 @@ # suspend/resume across pods shows the count continuing — proving the guest # memory snapshot round-tripped. The ActorTemplate is not a CRD: it lives in # counter-microvm-template.yaml.tmpl and is created through the ate API with -# `kubectl ate create actor-template`. The pool follows the microvm class +# `kubectl ate create actor-template`. The pool follows the microVM class # default SandboxConfig, which hack/install-microvm-deps.sh installs. apiVersion: v1 diff --git a/demos/egress/egress-microvm.yaml.tmpl b/demos/egress/egress-microvm.yaml.tmpl index 66c75b06c4..e1ac4ead12 100644 --- a/demos/egress/egress-microvm.yaml.tmpl +++ b/demos/egress/egress-microvm.yaml.tmpl @@ -17,7 +17,7 @@ # sandbox classes. The ActorTemplate is a substrate resource, not a CRD: it # lives in egress-microvm-template.yaml.tmpl and is created through the # ate API with `kubectl ate create actor-template`. The pool follows the -# microvm class default SandboxConfig, which hack/install-microvm-deps.sh +# microVM class default SandboxConfig, which hack/install-microvm-deps.sh # installs. apiVersion: v1 diff --git a/docs/dev/microvm-local.md b/docs/dev/microvm-local.md index 08b897ab98..545773f301 100644 --- a/docs/dev/microvm-local.md +++ b/docs/dev/microvm-local.md @@ -60,7 +60,7 @@ This is a one-shot bring-up: it deploys the control plane, installs the cluster-wide microVM deps via `hack/install-microvm-deps.sh` — assembling the guest runtime assets for your architecture (skipped if already present under `bin/microvm-assets/`), staging them into the in-cluster rustfs bucket, and -creating the microvm class default `SandboxConfig` — then deploys the demo worker pool + +creating the microVM class default `SandboxConfig` — then deploys the demo worker pool + template. ### 4. Verify diff --git a/docs/upgrade.md b/docs/upgrade.md index 928760b3af..94937452ef 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -220,9 +220,15 @@ go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml This applies the SandboxConfig admission policy and creates this release's gVisor SandboxConfig, `substrate-$NEW_RELEASE-gvisor`, as the gVisor class default. The old release's config stays in the cluster. gVisor pools without a `configRef` move to the new config without rolling their workers. New actors start on its default version, and actors suspended on a version it still lists restore on it. Pools pinned to a config with `configRef` stay on it until you change their `configRef`. -To create the config without moving any pool, add `--add-default-annotation=false`. Later, make it the default with `kubectl annotate sandboxconfig substrate-$NEW_RELEASE-gvisor sandboxconfig.ate.dev/is-class-default=true`. +To create the config without moving any pool, add `--add-default-annotation=false`, and pass it to `deploy ate-system` in step 4e too. Later, make it the default with `kubectl annotate sandboxconfig substrate-$NEW_RELEASE-gvisor sandboxconfig.ate.dev/is-class-default=true`. -If the cluster runs microVM pools, also run `hack/install-microvm-deps.sh --install` from the new checkout. It creates `substrate-$NEW_RELEASE-microvm` the same way and takes the same `--add-default-annotation` flag. It first stages the assets at fixed paths in the bucket. If the release changes them, earlier microVM configs no longer match the bucket, and atelet cannot fetch their assets on nodes that have not cached them. MicroVM pools pinned to an earlier config, or left on one by `--add-default-annotation=false`, then need to move to the new config. +If the cluster runs microVM pools, also run this from the new checkout: + +```bash +BUCKET_NAME=$(sed -n 's/^ *bucketName: //p' ~/ate-upgrade/new.yaml) VERSION=$NEW_RELEASE hack/install-microvm-deps.sh --install +``` + +The script reads neither `new.yaml` nor the image tag, so the variables give it the bucket and the release. It creates `substrate-$NEW_RELEASE-microvm` the same way and takes the same `--add-default-annotation` flag. It first stages the assets at fixed paths in the bucket. If the release changes them, earlier microVM configs no longer match the bucket, and atelet cannot fetch their assets on nodes that have not cached them. MicroVM pools pinned to an earlier config, or left on one by `--add-default-annotation=false`, then need to move to the new config. - **Done:** the command exits 0. - **Stuck:** the command fails. [Roll back step 4](#roll-back-step-4). @@ -234,7 +240,7 @@ If the cluster runs microVM pools, also run `hack/install-microvm-deps.sh --inst go run ./cmd/ate-setup deploy ate-system --config ~/ate-upgrade/new.yaml --setup-csi=none ``` -This applies what the steps above do not, such as the ate-system NetworkPolicies and the bundled PostgreSQL. The components above are already on the new release and do not roll again. `--setup-csi=none` leaves the CSI drivers as they are. Rerunning is safe. +This applies what the steps above do not, such as the ate-system NetworkPolicies and the bundled PostgreSQL. The components above are already on the new release and do not roll again. `--setup-csi=none` leaves the CSI drivers as they are. Rerunning is safe. If you passed `--add-default-annotation=false` in step 4d, add it here too, or this step annotates `substrate-$NEW_RELEASE-gvisor` and moves the gVisor pools. - **Done:** the command exits 0. - **Stuck:** the command fails. [Roll back step 4](#roll-back-step-4). @@ -273,15 +279,19 @@ Tell actor owners to stop using API fields new in this release, because the old ```bash ROLLBACK=substrate-$NEW_RELEASE-gvisor-rollback -sed "s/\${SANDBOXCONFIG_NAME}/$ROLLBACK/" manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl | kubectl create -f - -kubectl annotate sandboxconfig $ROLLBACK sandboxconfig.ate.dev/is-class-default=true +sed "s/\${SANDBOXCONFIG_NAME}/$ROLLBACK/" manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl | kubectl apply -f - +kubectl annotate --overwrite sandboxconfig $ROLLBACK sandboxconfig.ate.dev/is-class-default=true go run ./cmd/ate-setup deploy atenet --config ~/ate-upgrade/old.yaml go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/old.yaml go run ./cmd/ate-setup deploy apiserver --config ~/ate-upgrade/old.yaml go install ./cmd/kubectl-ate ``` -The first two `kubectl` commands create a new gVisor SandboxConfig from the old release's template and make it the class default. It is the newest default, so gVisor pools without a `configRef` move to it. Running `deploy sandboxconfig` from the old checkout does not do this. The old release's config already exists and is older than the new one, so it stays behind the new default even with the annotation, and `deploy sandboxconfig` warns about it. Actors suspended on a version the rollback config does not list resume without their memory. To keep restoring snapshots taken on a version new in this release, copy that version's entry from the new checkout's template into the rollback config before you create it. Pools pinned to `substrate-$NEW_RELEASE-gvisor` stay on it until you change their `configRef`. Roll back microVM pools the same way, from `manifests/microvm/sandboxconfig-microvm.yaml.tmpl`, with `${BUCKET_SCHEME}` and `${ASSET_BUCKET_NAME}` filled in as `hack/install-microvm-deps.sh` fills them. If the release changed the microVM assets, first stage the old ones from the old checkout with `hack/microvm-assets/assemble.sh` and the stage script, as `hack/install-microvm-deps.sh --install` runs them. +If `kubectl get sandboxconfig substrate-$NEW_RELEASE-gvisor` finds no config, step 4d did not create it, so skip the first three lines. The old release's config is still the class default. + +The two `kubectl` commands create a new gVisor SandboxConfig from the old release's template and make it the class default. It is the newest default, so gVisor pools without a `configRef` move to it. Running `deploy sandboxconfig` from the old checkout does not do this. The old release's config already exists and is older than the new one, so it stays behind the new default even with the annotation, and `deploy sandboxconfig` warns about it. Actors suspended on a version the rollback config does not list resume without their memory. To keep restoring snapshots taken on a version new in this release, copy that version's entry from the new checkout's template into the rollback config before you create it. Pools pinned to `substrate-$NEW_RELEASE-gvisor` stay on it until you change their `configRef`. Roll back microVM pools the same way, from `manifests/microvm/sandboxconfig-microvm.yaml.tmpl`, with `${BUCKET_SCHEME}` and `${ASSET_BUCKET_NAME}` filled in as `hack/install-microvm-deps.sh` fills them. If the release changed the microVM assets, first stage the old ones from the old checkout with `hack/microvm-assets/assemble.sh` and the stage script, as `hack/install-microvm-deps.sh --install` runs them. + +If you later upgrade to the same release again, the rollback config is newer than `substrate-$NEW_RELEASE-gvisor` and stays the class default. In step 4d, after `deploy sandboxconfig`, remove its annotation with `kubectl annotate sandboxconfig substrate-$NEW_RELEASE-gvisor-rollback sandboxconfig.ate.dev/is-class-default-`. Do the same for a microVM rollback config. `deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. No pool rolls. diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index c300a8e466..a69021a7ee 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Install the cluster-wide micro-VM (kata + cloud-hypervisor) dependencies. +# Install the cluster-wide microVM (kata + cloud-hypervisor) dependencies. # This is opt-in on top of ate-setup deploy ate-system, which only installs # the gVisor SandboxConfig. Used by: # * hack/run-microvm-demo.sh (before applying the counter-microvm demo) @@ -71,8 +71,8 @@ usage() { Usage: $0 --install [--add-default-annotation=true|false] Options: - --install Assemble + stage micro-VM assets and create this - release's microvm SandboxConfig, + --install Assemble + stage microVM assets and create this + release's microVM SandboxConfig, substrate--microvm. --add-default-annotation With --install, add the class-default annotation to the config (default: true). @@ -135,12 +135,16 @@ fi DEFAULT_ANNOTATION="sandboxconfig.ate.dev/is-class-default" -# One line per microvm SandboxConfig: +# Prints one line per microVM SandboxConfig: # name|creationTimestamp|default annotation|deletionTimestamp. +get_microvm_configs() { + run_kubectl get sandboxconfigs \ + -o jsonpath='{range .items[?(@.spec.sandboxClass=="microvm")]}{.metadata.name}{"|"}{.metadata.creationTimestamp}{"|"}{.metadata.annotations.sandboxconfig\.ate\.dev/is-class-default}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' +} + # This fails before the slow assemble and stage if the control plane, and # with it the SandboxConfig CRD, is not installed. -microvm_configs="$(run_kubectl get sandboxconfigs \ - -o jsonpath='{range .items[?(@.spec.sandboxClass=="microvm")]}{.metadata.name}{"|"}{.metadata.creationTimestamp}{"|"}{.metadata.annotations.sandboxconfig\.ate\.dev/is-class-default}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}')" +microvm_configs="$(get_microvm_configs)" # --- install ---------------------------------------------------------------- @@ -157,43 +161,45 @@ if [[ ! "${SANDBOXCONFIG_NAME}" =~ ^${dns_label}(\.${dns_label})*$ || ${#SANDBOX exit 1 fi -# Check the config before the slow assemble and stage, as -# `ate-setup deploy sandboxconfig` checks the gVisor one: -# * a config of this name must not be being deleted -# * with --add-default-annotation, warn when a newer config has the -# annotation. The newest one is the class default, so this one would not be. -# * without it, warn when no config of the class has the annotation +# check_configs checks this release's config against the microVM configs in +# $1, as `ate-setup deploy sandboxconfig` checks the gVisor one: +# * it exits when the config is being deleted +# * it sets warning when the config has the annotation but a newer config +# has it too. The newest one is the class default, so this one is not. +# * it sets warning when no config of the class has the annotation # Names compare byte by byte, as atecontroller compares them. -export LC_ALL=C -existing_created="" -newest_name="" -newest_created="" -while IFS='|' read -r name created is_default deleting; do - [[ -n "${name}" ]] || continue - if [[ "${name}" == "${SANDBOXCONFIG_NAME}" ]]; then - existing_created="${created}" - if [[ -n "${deleting}" ]]; then - echo "Error: SandboxConfig ${SANDBOXCONFIG_NAME} is being deleted; wait until it" >&2 - echo " is gone, then run this again." >&2 - exit 1 +check_configs() { + local name created is_default deleting + local existing_default="" newest_name="" newest_created="" + while IFS='|' read -r name created is_default deleting; do + [[ -n "${name}" ]] || continue + if [[ "${name}" == "${SANDBOXCONFIG_NAME}" ]]; then + existing_default="${is_default}" + if [[ -n "${deleting}" ]]; then + echo "Error: SandboxConfig ${SANDBOXCONFIG_NAME} is being deleted; wait until it" >&2 + echo " is gone, then run this again." >&2 + exit 1 + fi fi + [[ "${is_default}" == "true" && -z "${deleting}" ]] || continue + # RFC 3339 timestamps in UTC sort as strings. + if [[ -z "${newest_name}" || "${created}" > "${newest_created}" ]] || + [[ "${created}" == "${newest_created}" && "${name}" < "${newest_name}" ]]; then + newest_name="${name}" + newest_created="${created}" + fi + done <<<"$1" + warning="" + if [[ -z "${newest_name}" ]]; then + warning="no microVM SandboxConfig has the ${DEFAULT_ANNOTATION} annotation, so microVM WorkerPools without a configRef resolve no config." + elif [[ "${existing_default}" == "true" && "${newest_name}" != "${SANDBOXCONFIG_NAME}" ]]; then + warning="SandboxConfig ${SANDBOXCONFIG_NAME} has the ${DEFAULT_ANNOTATION} annotation but is not the microVM class default, because ${newest_name} is newer." fi - [[ "${is_default}" == "true" && -z "${deleting}" ]] || continue - # RFC 3339 timestamps in UTC sort as strings. - if [[ -z "${newest_name}" || "${created}" > "${newest_created}" ]] || - [[ "${created}" == "${newest_created}" && "${name}" < "${newest_name}" ]]; then - newest_name="${name}" - newest_created="${created}" - fi -done <<<"${microvm_configs}" -warning="" -if [[ "${add_default_annotation}" == "false" && -z "${newest_name}" ]]; then - warning="no microvm SandboxConfig has the ${DEFAULT_ANNOTATION} annotation, so microvm WorkerPools without a configRef resolve no config." -elif [[ "${add_default_annotation}" == "true" && -n "${existing_created}" && -n "${newest_name}" && "${newest_name}" != "${SANDBOXCONFIG_NAME}" ]] && - { [[ "${newest_created}" > "${existing_created}" ]] || - [[ "${newest_created}" == "${existing_created}" && "${newest_name}" < "${SANDBOXCONFIG_NAME}" ]]; }; then - warning="SandboxConfig ${SANDBOXCONFIG_NAME} has the ${DEFAULT_ANNOTATION} annotation but is not the microvm class default, because ${newest_name} is newer." -fi +} +export LC_ALL=C +# Checked before the slow assemble and stage. The warning is about the +# cluster before the apply. +check_configs "${microvm_configs}" # Target arch: match the images' platform (KO_DEFAULTPLATFORMS is set by # .ate-dev-env.sh on GKE and by the kind wrapper); fall back to the host arch. @@ -246,7 +252,7 @@ else OUT="${OUT}" BUCKET="${ASSET_BUCKET_NAME}" hack/microvm-assets/stage-to-gcs.sh fi -# --- 3. create this release's microvm SandboxConfig ------------------------- +# --- 3. create this release's microVM SandboxConfig ------------------------- # Every asset is downloaded rather than built, so all four carry committed, # reproducible per-arch shas and the name and the bucket's scheme and name are # the only substitutions left. @@ -262,6 +268,9 @@ sed -e "s|\${SANDBOXCONFIG_NAME}|${SANDBOXCONFIG_NAME}|g" \ if [[ "${add_default_annotation}" == "true" ]]; then run_kubectl annotate --overwrite sandboxconfig "${SANDBOXCONFIG_NAME}" "${DEFAULT_ANNOTATION}=true" fi +# Checked again against the config's real creation time and annotations. +microvm_configs="$(get_microvm_configs)" +check_configs "${microvm_configs}" if [[ -n "${warning}" ]]; then echo "Warning: ${warning}" >&2 fi diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 9ba36ad330..c667ed1bf6 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -20,7 +20,7 @@ # # Composes: # 1. ate-setup deploy ate-system (control plane) -# 2. hack/install-microvm-deps.sh --install (asset build/stage + the microvm +# 2. hack/install-microvm-deps.sh --install (asset build/stage + the microVM # class default SandboxConfig) # 3. Deploy the counter-microvm demo (worker pool manifest, atespace, and # ActorTemplate through the ate API). From 83d13ecad6c4f069d553e93113b02fd48a147e73 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Tue, 27 Oct 2026 07:55:24 -0700 Subject: [PATCH 4/9] docs: move SandboxConfig upgrade steps to their own guide docs/sandboxconfig-upgrade.md covers creating a release's SandboxConfigs, holding pools back with --add-default-annotation=false, and rollback. Step 4d of the runbook keeps the default command and links to it. The old release's config keeps its class-default annotation through the upgrade. Rollback now removes the annotation from the new release's config, so the old one is the class default again, instead of creating a separate rollback config. Signed-off-by: zoezhao --- README.md | 1 + docs/sandboxconfig-upgrade.md | 61 +++++++++++++++++++++++++++++++++++ docs/upgrade.md | 27 +++------------- docs/version-skew-policy.md | 4 +-- 4 files changed, 69 insertions(+), 24 deletions(-) create mode 100644 docs/sandboxconfig-upgrade.md diff --git a/README.md b/README.md index 6e89e288e6..a0b691e924 100644 --- a/README.md +++ b/README.md @@ -274,6 +274,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * [Request Parking](docs/request-parking.md): How the router parks requests through transient worker-pool saturation. * [Sandbox Asset Cache](docs/sandbox-asset-cache.md): How a node caches the sandbox binaries actors run on, decides which runtimes it needs, and evicts to a budget. * [Upgrade Runbook](docs/upgrade.md): Upgrade a running install to a newer supported release, and roll back if needed. +* [SandboxConfig Upgrade](docs/sandboxconfig-upgrade.md): Move pools to a release's SandboxConfigs during an upgrade, and back on rollback. * [Node Storage](docs/node-storage.md): How to give snapshot I/O its own disk. * [Security Guide](docs/security.md): Security overview, security principles, and threat modeling. * [Benchmarking Guide](benchmarking/README.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness. diff --git a/docs/sandboxconfig-upgrade.md b/docs/sandboxconfig-upgrade.md new file mode 100644 index 0000000000..dea675ceb2 --- /dev/null +++ b/docs/sandboxconfig-upgrade.md @@ -0,0 +1,61 @@ +# SandboxConfig upgrade + +This guide covers the SandboxConfig part of an upgrade: step 4d of the [upgrade runbook](upgrade.md), and its rollback. Run it from the runbook's shell, with `NEW_RELEASE` set and `~/ate-upgrade` in place. + +Each release creates its own SandboxConfig per sandbox class, `substrate--gvisor` and `substrate--microvm`. Configs from earlier releases stay in the cluster. A WorkerPool without a `configRef` uses the newest config of its class with the `sandboxconfig.ate.dev/is-class-default` annotation. A pool with a `configRef` uses that config. [Class default](api-guide.md#class-default) in the API guide has the full rules. Moving a pool to another config does not roll its workers. + +New actors start on the default version of their pool's config. A suspended actor restores its memory only on a version with the same assets as the one its snapshot was taken on. If the pool's config has no such version, the actor resumes without its memory, keeping its files. The [version skew policy](version-skew-policy.md#sandboxconfig) says which versions a release keeps. + +## Upgrade + +### gVisor + +```bash +go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml +``` + +This applies the SandboxConfig admission policy and creates `substrate-$NEW_RELEASE-gvisor` as the gVisor class default. gVisor pools without a `configRef` move to it. Pools pinned to a config with `configRef` stay on it until you change their `configRef`. + +### microVM + +If the cluster runs microVM pools, also run this from the new checkout: + +```bash +BUCKET_NAME=$(sed -n 's/^ *bucketName: //p' ~/ate-upgrade/new.yaml) VERSION=$NEW_RELEASE hack/install-microvm-deps.sh --install +``` + +The script reads neither `new.yaml` nor the image tag, so the variables give it the bucket and the release. It creates `substrate-$NEW_RELEASE-microvm` the same way as the gVisor config. It first stages the assets at fixed paths in the bucket. If the release changes them, earlier microVM configs no longer match the bucket, and atelet cannot fetch their assets on nodes that have not cached them. MicroVM pools pinned to an earlier config, or left on one by `--add-default-annotation=false`, then need to move to the new config. + +### Create the configs without moving pools + +Add `--add-default-annotation=false` to both commands above. Pass it to `deploy ate-system` in step 4e of the runbook too, or that step adds the annotation and moves the gVisor pools. Later, make each config the class default: + +```bash +kubectl annotate sandboxconfig substrate-$NEW_RELEASE-gvisor sandboxconfig.ate.dev/is-class-default=true +kubectl annotate sandboxconfig substrate-$NEW_RELEASE-microvm sandboxconfig.ate.dev/is-class-default=true +``` + +### Check + +Each pool's `CONFIG` column shows the config it uses: + +```bash +kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,CLASS:.spec.sandboxClasses[0].name,CONFIG:.status.sandboxClasses[0].configRef.name' +``` + +## Roll back + +Roll back the SandboxConfigs first in the runbook's [Roll back step 4](upgrade.md#roll-back-step-4), from the old checkout. + +Actors suspended on a version the old release's config does not list resume without their memory. To keep restoring snapshots taken on a version new in this release, copy that version's entry from the new release's config into the old one before you go on. + +If the release changed the microVM assets, stage the old ones from the old checkout with `hack/microvm-assets/assemble.sh` and the stage script, as `hack/install-microvm-deps.sh --install` runs them. + +Then remove the class-default annotation from the new release's configs. Skip a line if the upgrade did not create that config. + +```bash +kubectl annotate sandboxconfig substrate-$NEW_RELEASE-gvisor sandboxconfig.ate.dev/is-class-default- +kubectl annotate sandboxconfig substrate-$NEW_RELEASE-microvm sandboxconfig.ate.dev/is-class-default- +``` + +The old release's config still has the annotation, so it is the class default again, and pools without a `configRef` move back to it. Pools pinned to a new release's config stay on it until you change their `configRef`. diff --git a/docs/upgrade.md b/docs/upgrade.md index 94937452ef..fafadcc737 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -218,20 +218,12 @@ kubectl -n ate-system rollout status deploy/atenet-egress go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml ``` -This applies the SandboxConfig admission policy and creates this release's gVisor SandboxConfig, `substrate-$NEW_RELEASE-gvisor`, as the gVisor class default. The old release's config stays in the cluster. gVisor pools without a `configRef` move to the new config without rolling their workers. New actors start on its default version, and actors suspended on a version it still lists restore on it. Pools pinned to a config with `configRef` stay on it until you change their `configRef`. +By default this creates this release's gVisor SandboxConfig, `substrate-$NEW_RELEASE-gvisor`, as the gVisor class default. gVisor pools without a `configRef` move to it without rolling their workers. If the cluster runs microVM pools, also follow [microVM](sandboxconfig-upgrade.md#microvm) in the SandboxConfig upgrade guide. -To create the config without moving any pool, add `--add-default-annotation=false`, and pass it to `deploy ate-system` in step 4e too. Later, make it the default with `kubectl annotate sandboxconfig substrate-$NEW_RELEASE-gvisor sandboxconfig.ate.dev/is-class-default=true`. +To control which pools move and when, and for what happens to suspended actors, see the [SandboxConfig upgrade guide](sandboxconfig-upgrade.md). -If the cluster runs microVM pools, also run this from the new checkout: - -```bash -BUCKET_NAME=$(sed -n 's/^ *bucketName: //p' ~/ate-upgrade/new.yaml) VERSION=$NEW_RELEASE hack/install-microvm-deps.sh --install -``` - -The script reads neither `new.yaml` nor the image tag, so the variables give it the bucket and the release. It creates `substrate-$NEW_RELEASE-microvm` the same way and takes the same `--add-default-annotation` flag. It first stages the assets at fixed paths in the bucket. If the release changes them, earlier microVM configs no longer match the bucket, and atelet cannot fetch their assets on nodes that have not cached them. MicroVM pools pinned to an earlier config, or left on one by `--add-default-annotation=false`, then need to move to the new config. - -- **Done:** the command exits 0. -- **Stuck:** the command fails. [Roll back step 4](#roll-back-step-4). +- **Done:** the commands exit 0. +- **Stuck:** a command fails. [Roll back step 4](#roll-back-step-4). - **Actors:** a suspended actor whose sandbox version the new config does not list resumes without its memory, keeping its files. #### 4e. The rest of the system bundle @@ -275,24 +267,15 @@ cd ~/ate-upgrade/old #### Roll back step 4 -Tell actor owners to stop using API fields new in this release, because the old API server rejects them. Then: +Tell actor owners to stop using API fields new in this release, because the old API server rejects them. Roll back the SandboxConfigs as in [Roll back](sandboxconfig-upgrade.md#roll-back) in the SandboxConfig upgrade guide. Then: ```bash -ROLLBACK=substrate-$NEW_RELEASE-gvisor-rollback -sed "s/\${SANDBOXCONFIG_NAME}/$ROLLBACK/" manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl | kubectl apply -f - -kubectl annotate --overwrite sandboxconfig $ROLLBACK sandboxconfig.ate.dev/is-class-default=true go run ./cmd/ate-setup deploy atenet --config ~/ate-upgrade/old.yaml go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/old.yaml go run ./cmd/ate-setup deploy apiserver --config ~/ate-upgrade/old.yaml go install ./cmd/kubectl-ate ``` -If `kubectl get sandboxconfig substrate-$NEW_RELEASE-gvisor` finds no config, step 4d did not create it, so skip the first three lines. The old release's config is still the class default. - -The two `kubectl` commands create a new gVisor SandboxConfig from the old release's template and make it the class default. It is the newest default, so gVisor pools without a `configRef` move to it. Running `deploy sandboxconfig` from the old checkout does not do this. The old release's config already exists and is older than the new one, so it stays behind the new default even with the annotation, and `deploy sandboxconfig` warns about it. Actors suspended on a version the rollback config does not list resume without their memory. To keep restoring snapshots taken on a version new in this release, copy that version's entry from the new checkout's template into the rollback config before you create it. Pools pinned to `substrate-$NEW_RELEASE-gvisor` stay on it until you change their `configRef`. Roll back microVM pools the same way, from `manifests/microvm/sandboxconfig-microvm.yaml.tmpl`, with `${BUCKET_SCHEME}` and `${ASSET_BUCKET_NAME}` filled in as `hack/install-microvm-deps.sh` fills them. If the release changed the microVM assets, first stage the old ones from the old checkout with `hack/microvm-assets/assemble.sh` and the stage script, as `hack/install-microvm-deps.sh --install` runs them. - -If you later upgrade to the same release again, the rollback config is newer than `substrate-$NEW_RELEASE-gvisor` and stays the class default. In step 4d, after `deploy sandboxconfig`, remove its annotation with `kubectl annotate sandboxconfig substrate-$NEW_RELEASE-gvisor-rollback sandboxconfig.ate.dev/is-class-default-`. Do the same for a microVM rollback config. - `deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. No pool rolls. Fields that the new release added to WorkerPools, SandboxConfigs or CSIDriverConfigs are dropped from those objects. Remove them from your YAML files too, or `kubectl apply` rejects the files. diff --git a/docs/version-skew-policy.md b/docs/version-skew-policy.md index 5dccd16aa8..e4c9ae3f23 100644 --- a/docs/version-skew-policy.md +++ b/docs/version-skew-policy.md @@ -82,9 +82,9 @@ Each release creates its own SandboxConfig per sandbox class, such as `substrate A kept version's assets must stay fetchable, so a release does not overwrite them. The microVM assets live at fixed paths in the cluster's bucket, so a microVM release that changes them replaces the version instead. Moving a pool to a new config does not change the pod template, so workers do not roll. -An actor that starts without memory state, new or cold booting, gets the default version. An actor restoring memory state gets a version with the same assets as the one its snapshot was taken on, and stays on it until it next starts without memory state. If the pool's config has no such version, the actor cold boots, keeping its files and losing its memory. Rollback creates a new config from the N template and makes it the class default, which removes the N+1 versions unless the rollback config keeps them. +An actor that starts without memory state, new or cold booting, gets the default version. An actor restoring memory state gets a version with the same assets as the one its snapshot was taken on, and stays on it until it next starts without memory state. If the pool's config has no such version, the actor cold boots, keeping its files and losing its memory. Rollback removes the class-default annotation from the N+1 config, so the previous default config before N+1 rollout is the class default again. That config does not list the sandbox versions new in N+1. An actor suspended on one of them resumes without its memory, unless you first add that version to the config. - +The [SandboxConfig upgrade guide](sandboxconfig-upgrade.md) covers the details. ### Database schema From cc6bb9bef52202dd6731efdad6786358a613b163 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Wed, 28 Oct 2026 16:02:56 -0700 Subject: [PATCH 5/9] atev1: share the class default rule atecontroller, ate-setup and the e2e preflight each picked the class default SandboxConfig with their own copy of the rule. ClassDefaultSandboxConfig now holds it, and all three call it. Signed-off-by: zoezhao --- api/infra/k8s/ate/v1/sandboxconfig_types.go | 19 +++++++++++ cmd/ate-setup/internal/steps/sandboxconfig.go | 22 +++---------- .../controllers/workerpool_sandboxconfig.go | 33 ++++--------------- internal/e2e/preflight.go | 6 +--- 4 files changed, 31 insertions(+), 49 deletions(-) diff --git a/api/infra/k8s/ate/v1/sandboxconfig_types.go b/api/infra/k8s/ate/v1/sandboxconfig_types.go index ab28685d44..e968804d07 100644 --- a/api/infra/k8s/ate/v1/sandboxconfig_types.go +++ b/api/infra/k8s/ate/v1/sandboxconfig_types.go @@ -212,6 +212,25 @@ type SandboxConfigList struct { Items []SandboxConfig `json:"items"` } +// ClassDefaultSandboxConfig returns the default SandboxConfig of class among +// configs, or nil if there is none. It is the newest config of the class with +// SandboxConfigClassDefaultAnnotation set to "true" that is not being deleted. +// Of two created in the same second, the one whose name sorts first wins. +func ClassDefaultSandboxConfig(configs []SandboxConfig, class SandboxClass) *SandboxConfig { + var best *SandboxConfig + for i := range configs { + sc := &configs[i] + if sc.Spec.SandboxClass != class || sc.Annotations[SandboxConfigClassDefaultAnnotation] != "true" || !sc.DeletionTimestamp.IsZero() { + continue + } + if best == nil || best.CreationTimestamp.Before(&sc.CreationTimestamp) || + (sc.CreationTimestamp.Equal(&best.CreationTimestamp) && sc.Name < best.Name) { + best = sc + } + } + return best +} + func init() { SchemeBuilder.Register(&SandboxConfig{}, &SandboxConfigList{}) } diff --git a/cmd/ate-setup/internal/steps/sandboxconfig.go b/cmd/ate-setup/internal/steps/sandboxconfig.go index 82ea34118f..be74fa6c52 100644 --- a/cmd/ate-setup/internal/steps/sandboxconfig.go +++ b/cmd/ate-setup/internal/steps/sandboxconfig.go @@ -163,17 +163,13 @@ func sandboxConfigName(version string, class atev1.SandboxClass) (string, error) // - It warns when the class has no default, so pools of the class without a // configRef resolve none. func checkClassDefault(configs []atev1.SandboxConfig, name string, class atev1.SandboxClass) (string, error) { - var existing, newest *atev1.SandboxConfig + var existing *atev1.SandboxConfig for i := range configs { - sc := &configs[i] - if sc.Name == name { - existing = sc - } - if sc.Spec.SandboxClass == class && sc.Annotations[atev1.SandboxConfigClassDefaultAnnotation] == "true" && sc.DeletionTimestamp.IsZero() && - (newest == nil || newerSandboxConfig(sc, newest)) { - newest = sc + if configs[i].Name == name { + existing = &configs[i] } } + newest := atev1.ClassDefaultSandboxConfig(configs, class) if existing != nil && !existing.DeletionTimestamp.IsZero() { return "", fmt.Errorf("SandboxConfig %s is being deleted; wait until it is gone, then run this again", name) } @@ -187,13 +183,3 @@ func checkClassDefault(configs []atev1.SandboxConfig, name string, class atev1.S } return "", nil } - -// newerSandboxConfig reports whether a takes precedence over b as a class -// default: it was created later, or in the same second with a smaller name. -// atecontroller applies the same rule. -func newerSandboxConfig(a, b *atev1.SandboxConfig) bool { - if !a.CreationTimestamp.Equal(&b.CreationTimestamp) { - return b.CreationTimestamp.Before(&a.CreationTimestamp) - } - return a.Name < b.Name -} diff --git a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig.go b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig.go index 6227f4585d..af87fdf25a 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig.go +++ b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig.go @@ -31,9 +31,8 @@ import ( // // - With configRef, that config is used. It must exist, belong to the same // class, and not be being deleted. -// - Without configRef, the newest config of the class that is annotated as -// the class default and is not being deleted is used. If two were created -// at the same time, the name that sorts first wins. +// - Without configRef, the class default from +// atev1.ClassDefaultSandboxConfig is used. func resolveLatestSandboxConfig(entry atev1.WorkerPoolSandboxClass, configs []atev1.SandboxConfig) (*atev1.SandboxConfig, string, string) { if entry.ConfigRef != nil { name := entry.ConfigRef.Name @@ -54,22 +53,13 @@ func resolveLatestSandboxConfig(entry atev1.WorkerPoolSandboxClass, configs []at return sc, "", "" } - isDefault := func(sc atev1.SandboxConfig) bool { - return sc.Spec.SandboxClass == entry.Name && - sc.Annotations[atev1.SandboxConfigClassDefaultAnnotation] == "true" - } - var best *atev1.SandboxConfig - for i := range configs { - sc := &configs[i] - if isDefault(*sc) && sc.DeletionTimestamp.IsZero() && (best == nil || newerSandboxConfig(sc, best)) { - best = sc - } - } - if best != nil { - return best, "", "" + if sc := atev1.ClassDefaultSandboxConfig(configs, entry.Name); sc != nil { + return sc, "", "" } isDeletingDefault := func(sc atev1.SandboxConfig) bool { - return isDefault(sc) && !sc.DeletionTimestamp.IsZero() + return sc.Spec.SandboxClass == entry.Name && + sc.Annotations[atev1.SandboxConfigClassDefaultAnnotation] == "true" && + !sc.DeletionTimestamp.IsZero() } if slices.ContainsFunc(configs, isDeletingDefault) { return nil, atev1.WorkerPoolReasonSandboxConfigDeleting, @@ -122,15 +112,6 @@ func keepSandboxConfig(class atev1.SandboxClass, inUse string, configs []atev1.S } } -// newerSandboxConfig reports whether a takes precedence over b as a class -// default. -func newerSandboxConfig(a, b *atev1.SandboxConfig) bool { - if !a.CreationTimestamp.Equal(&b.CreationTimestamp) { - return b.CreationTimestamp.Before(&a.CreationTimestamp) - } - return a.Name < b.Name -} - // resolveSandboxClasses resolves every entry of wp.Spec.SandboxClasses. It // returns one status entry per class, and whether every class resolved. Each // entry carries the config the class uses, nil if it has none, and its diff --git a/internal/e2e/preflight.go b/internal/e2e/preflight.go index 22f5c20d8a..32a913a95d 100644 --- a/internal/e2e/preflight.go +++ b/internal/e2e/preflight.go @@ -17,7 +17,6 @@ package e2e import ( "context" "fmt" - "slices" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,10 +72,7 @@ func PreflightChecks() error { if err != nil { return fmt.Errorf("failed to list SandboxConfigs: %w", err) } - if !slices.ContainsFunc(list.Items, func(sc atev1.SandboxConfig) bool { - return sc.Spec.SandboxClass == atev1.SandboxClassMicroVM && - sc.Annotations[atev1.SandboxConfigClassDefaultAnnotation] == "true" && sc.DeletionTimestamp.IsZero() - }) { + if atev1.ClassDefaultSandboxConfig(list.Items, atev1.SandboxClassMicroVM) == nil { return fmt.Errorf("E2E_SANDBOX_CLASS=%s but no SandboxConfig of the class is annotated %s=true (see docs/dev/microvm-local.md for cluster setup)", SandboxClassMicroVM, atev1.SandboxConfigClassDefaultAnnotation) } From 7213ee138c5fe6ce9d4afb57fa6cdf71af9e571f Mon Sep 17 00:00:00 2001 From: zoezhao Date: Wed, 28 Oct 2026 16:03:19 -0700 Subject: [PATCH 6/9] ate-setup: name dev builds by hash and leave existing configs alone Naming every build's SandboxConfig after its version gave each dev commit a new class default, and gave a moving tag like main one config updated in place. Only a release tag now names the config after the version. Any other build gets substrate-dev--, hashed from the rendered config, so builds with the same config share one. ate-setup and install-microvm-deps.sh create the config only when it does not exist, so a rerun keeps edits such as a disabled version. They leave the annotations of an existing config alone too. --add-default-annotation becomes --create-with-default-annotation. It defaults to true and applies only to a config the run creates. deploy ate-system computes the config once and passes it to deploySandboxConfig. The templates go back to the version names on main. Signed-off-by: zoezhao --- .../ate/v1/sandboxconfig_validation_test.go | 2 +- cmd/ate-setup/commands.md | 10 +- cmd/ate-setup/internal/cmd/deploy.go | 43 ++-- cmd/ate-setup/internal/images/release.go | 3 +- cmd/ate-setup/internal/steps/deploy.go | 14 +- cmd/ate-setup/internal/steps/sandboxconfig.go | 196 +++++++++++------- .../internal/steps/sandboxconfig_test.go | 81 +++++--- demos/counter/counter.yaml.tmpl | 2 +- docs/api-guide.md | 8 +- docs/sandboxconfig-upgrade.md | 4 +- docs/upgrade.md | 2 +- hack/install-microvm-deps.sh | 92 ++++---- .../sandboxconfig-gvisor.yaml.tmpl | 11 +- .../microvm/sandboxconfig-microvm.yaml.tmpl | 12 +- 14 files changed, 285 insertions(+), 195 deletions(-) diff --git a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go index 1e52ff7ad6..03d5d3cb91 100644 --- a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go +++ b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go @@ -366,7 +366,7 @@ func TestShippedSandboxConfigManifests(t *testing.T) { if err := yaml.UnmarshalStrict(raw, sc); err != nil { t.Fatalf("decode manifest: %v", err) } - // The installers add the annotation only with --add-default-annotation. + // The installers add the annotation when they create the config. if _, ok := sc.Annotations[SandboxConfigClassDefaultAnnotation]; ok { t.Errorf("template sets %s; the installers add it", SandboxConfigClassDefaultAnnotation) } diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index c685a3bc1c..8288e5bf3f 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -113,10 +113,12 @@ apiserver, the controller, atenet, and atelet. It creates every `create` resource below on the way, so those subcommands are only needed to redo one on a running cluster. -`deploy ate-system` and `deploy sandboxconfig` create this release's gVisor -SandboxConfig, `substrate--gvisor`, with the -`sandboxconfig.ate.dev/is-class-default` annotation. Pass -`--add-default-annotation=false` to leave the annotation off. +`deploy ate-system` and `deploy sandboxconfig` create this build's gVisor +SandboxConfig with the `sandboxconfig.ate.dev/is-class-default` annotation. A +release names it `substrate--gvisor`. Any other build names it +`substrate-dev--gvisor`, hashed from the rendered config. Pass +`--create-with-default-annotation=false` to create it without the annotation. +An existing config is left as it is, annotations included. ## Publish diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index dafe99e143..0869b9b6f7 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -84,7 +84,7 @@ Shape the install with the global --atenet-dataplane, --cluster-size, if err != nil { return err } - if opts.AddSandboxConfigDefaultAnnotation, err = cmd.Flags().GetBool(addDefaultAnnotationFlag); err != nil { + if opts.CreateSandboxConfigWithDefaultAnnotation, err = cmd.Flags().GetBool(createWithDefaultAnnotationFlag); err != nil { return err } return env.DeployAteSystem(cmd.Context(), opts) @@ -139,31 +139,36 @@ var deployPodCertControllerCmd = &cobra.Command{ }, } -// addDefaultAnnotationFlag names the flag that adds the class-default -// annotation to the release's gVisor SandboxConfig. It is a plain flag rather -// than a recorded setting: each run decides it anew. -const addDefaultAnnotationFlag = "add-default-annotation" +// createWithDefaultAnnotationFlag names the flag that leaves the +// class-default annotation off this build's gVisor SandboxConfig when false. +// It is a plain flag rather than a recorded setting, because it applies only +// to the run that creates the config. +const createWithDefaultAnnotationFlag = "create-with-default-annotation" -const addDefaultAnnotationUsage = "Add the sandboxconfig.ate.dev/is-class-default annotation to this release's gVisor SandboxConfig" +const createWithDefaultAnnotationUsage = "Add the sandboxconfig.ate.dev/is-class-default annotation to this build's gVisor SandboxConfig if this run creates it. An existing config is left as it is" var deploySandboxConfigCmd = &cobra.Command{ Use: "sandboxconfig", - Short: "Deploy the SandboxConfig admission policy and this release's gVisor SandboxConfig only", - Long: `Deploy the SandboxConfig admission policy and this release's gVisor -SandboxConfig, substrate--gvisor, rendered from -manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl. - -With --add-default-annotation (the default) the config gets the -sandboxconfig.ate.dev/is-class-default annotation. Every gVisor WorkerPool -without a configRef follows the newest config with the annotation, so a -warning names a newer one if it exists.`, + Short: "Deploy the SandboxConfig admission policy and this build's gVisor SandboxConfig only", + Long: `Deploy the SandboxConfig admission policy and create this build's gVisor +SandboxConfig from manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl. A +release names it substrate--gvisor. Any other build names it +substrate-dev--gvisor, hashed from the rendered config, so builds with +the same config share one. + +When this run creates the config, it gets the +sandboxconfig.ate.dev/is-class-default annotation, unless +--create-with-default-annotation=false. An existing config is left as it is, +annotations included. Change its annotation with kubectl annotate. Every +gVisor WorkerPool without a configRef follows the newest config with the +annotation, so a warning names a newer one if it exists.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - addDefaultAnnotation, err := cmd.Flags().GetBool(addDefaultAnnotationFlag) + create, err := cmd.Flags().GetBool(createWithDefaultAnnotationFlag) if err != nil { return err } - return env.DeploySandboxConfig(cmd.Context(), addDefaultAnnotation) + return env.DeploySandboxConfig(cmd.Context(), create) }, } @@ -203,6 +208,6 @@ func init() { }) config.BindCommandFlags(deployAteSystemPath, deployAteSystemCmd.Flags()) deployAteSystemCmd.Flags().Lookup("setup-csi").NoOptDefVal = "none" - deployAteSystemCmd.Flags().Bool(addDefaultAnnotationFlag, true, addDefaultAnnotationUsage) - deploySandboxConfigCmd.Flags().Bool(addDefaultAnnotationFlag, true, addDefaultAnnotationUsage) + deployAteSystemCmd.Flags().Bool(createWithDefaultAnnotationFlag, true, createWithDefaultAnnotationUsage) + deploySandboxConfigCmd.Flags().Bool(createWithDefaultAnnotationFlag, true, createWithDefaultAnnotationUsage) } diff --git a/cmd/ate-setup/internal/images/release.go b/cmd/ate-setup/internal/images/release.go index 75d30f1213..7f56370a0a 100644 --- a/cmd/ate-setup/internal/images/release.go +++ b/cmd/ate-setup/internal/images/release.go @@ -48,7 +48,8 @@ const RevisionLabel = "org.opencontainers.image.revision" const VersionLabel = "org.opencontainers.image.version" // releaseTag matches a release tag, vMAJOR.MINOR.PATCH, or a release candidate, -// vMAJOR.MINOR.PATCH-PRERELEASE. hack/release/draft-release.sh accepts the same. +// vMAJOR.MINOR.PATCH-PRERELEASE. hack/release/draft-release.sh and +// hack/install-microvm-deps.sh accept the same. var releaseTag = regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$`) // CheckReleaseVersion returns an error unless version is a release tag. diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index c9b302bddc..03f07997d0 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -43,9 +43,9 @@ type DeployOptions struct { // SetupCSI additionally installs the CSI driver (nfs, hostpath, both, none). // Kind only. The hostpath driver is Kind only. SetupCSI string - // AddSandboxConfigDefaultAnnotation adds the class-default annotation to - // the release's gVisor SandboxConfig. See DeploySandboxConfig. - AddSandboxConfigDefaultAnnotation bool + // CreateSandboxConfigWithDefaultAnnotation adds the class-default + // annotation to this build's gVisor SandboxConfig if the deploy creates it. + CreateSandboxConfigWithDefaultAnnotation bool } // Validate checks the options that can be checked without configuration or a @@ -89,9 +89,9 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err := e.requireNoWorkerPods(ctx); err != nil { return err } - // DeploySandboxConfig checks the config again partway through. Checking - // here keeps a refusal from leaving a half-upgraded system. - if _, err := e.checkSandboxConfig(ctx); err != nil { + // Checked here so a refusal does not leave a half-upgraded system. + sc, err := e.checkSandboxConfig(ctx) + if err != nil { return err } @@ -120,7 +120,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } - if err := e.DeploySandboxConfig(ctx, opts.AddSandboxConfigDefaultAnnotation); err != nil { + if err := e.deploySandboxConfig(ctx, sc, opts.CreateSandboxConfigWithDefaultAnnotation); err != nil { return err } diff --git a/cmd/ate-setup/internal/steps/sandboxconfig.go b/cmd/ate-setup/internal/steps/sandboxconfig.go index be74fa6c52..7d7ae5874f 100644 --- a/cmd/ate-setup/internal/steps/sandboxconfig.go +++ b/cmd/ate-setup/internal/steps/sandboxconfig.go @@ -16,8 +16,12 @@ package steps import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" + "regexp" + "slices" "strings" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -28,16 +32,29 @@ import ( "k8s.io/client-go/dynamic" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/cmd/ate-setup/internal/log" "go.ate.dev/substrate/cmd/ate-setup/internal/render" ) -// DeploySandboxConfig applies the SandboxConfig admission policy, then this -// release's gVisor SandboxConfig, substrate--gvisor, rendered from -// sandboxconfig-gvisor.yaml.tmpl. With addDefaultAnnotation it also adds the -// class-default annotation to the config. -func (e *Env) DeploySandboxConfig(ctx context.Context, addDefaultAnnotation bool) error { +// DeploySandboxConfig applies the SandboxConfig admission policy, then +// creates this build's gVisor SandboxConfig from sandboxconfig-gvisor.yaml.tmpl +// unless it exists. sandboxConfigName says how the config is named. A config +// it creates gets the class-default annotation when +// createWithDefaultAnnotation is true. An existing config, annotations +// included, is left as it is. +func (e *Env) DeploySandboxConfig(ctx context.Context, createWithDefaultAnnotation bool) error { + sc, err := e.checkSandboxConfig(ctx) + if err != nil { + return err + } + return e.deploySandboxConfig(ctx, sc, createWithDefaultAnnotation) +} + +// deploySandboxConfig is DeploySandboxConfig for the config that +// checkSandboxConfig returned. +func (e *Env) deploySandboxConfig(ctx context.Context, sc sandboxConfig, createWithDefaultAnnotation bool) error { log.Step("deploy_sandboxconfig") if err := e.EnsureCRDs(ctx); err != nil { @@ -50,64 +67,81 @@ func (e *Env) DeploySandboxConfig(ctx context.Context, addDefaultAnnotation bool return err } - name, err := e.checkSandboxConfig(ctx) - if err != nil { - return err - } - - manifest, err := render.Template(e.Cfg.Manifest("sandboxconfig-gvisor.yaml.tmpl"), map[string]string{"SANDBOXCONFIG_NAME": name}, nil) - if err != nil { - return err - } - if err := e.Kube.ApplyBytes(ctx, manifest); err != nil { - return err - } - log.Infof("applied SandboxConfig %s", name) - if addDefaultAnnotation { - // A merge patch rather than part of the apply above. The apply then - // never owns the annotation, so a later run without - // --add-default-annotation leaves it in place. - patch, err := json.Marshal(map[string]any{ - "metadata": map[string]any{"annotations": map[string]any{atev1.SandboxConfigClassDefaultAnnotation: "true"}}, - }) - if err != nil { - return fmt.Errorf("while building the patch for SandboxConfig %s: %w", name, err) + if sc.exists { + // An existing config may have been edited since, so it is left as it + // is. + log.Infof("SandboxConfig %s exists; leaving it and its annotations as they are", sc.name) + } else { + if err := e.Kube.ApplyBytes(ctx, sc.manifest); err != nil { + return err } - if _, err := e.sandboxConfigs().Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{FieldManager: kube.FieldManager}); err != nil { - return fmt.Errorf("while annotating SandboxConfig %s: %w", name, err) + log.Infof("created SandboxConfig %s", sc.name) + if createWithDefaultAnnotation { + patch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{"annotations": map[string]any{atev1.SandboxConfigClassDefaultAnnotation: "true"}}, + }) + if err != nil { + return fmt.Errorf("while building the patch for SandboxConfig %s: %w", sc.name, err) + } + if _, err := e.sandboxConfigs().Patch(ctx, sc.name, types.MergePatchType, patch, metav1.PatchOptions{FieldManager: kube.FieldManager}); err != nil { + return fmt.Errorf("while annotating SandboxConfig %s: %w", sc.name, err) + } + log.Infof("annotated SandboxConfig %s %s=true", sc.name, atev1.SandboxConfigClassDefaultAnnotation) } - log.Infof("annotated SandboxConfig %s %s=true", name, atev1.SandboxConfigClassDefaultAnnotation) } - // Checked after the apply, against the config's real creation time and + // Checked after the create, against the config's real creation time and // annotations. configs, err := e.listSandboxConfigs(ctx) if err != nil { return err } - warning, err := checkClassDefault(configs, name, atev1.SandboxClassGvisor) - if warning != "" { + if warning := classDefaultWarning(configs, sc.name, atev1.SandboxClassGvisor); warning != "" { log.Warnf("%s", warning) } - return err + return nil } -// checkSandboxConfig returns the name of this release's gVisor SandboxConfig. -// It fails when checkClassDefault does. -func (e *Env) checkSandboxConfig(ctx context.Context) (string, error) { - class := atev1.SandboxClassGvisor - name, err := sandboxConfigName(e.SubstrateVersion(), class) +// sandboxConfig is this build's gVisor SandboxConfig. +type sandboxConfig struct { + name string + // manifest is the config rendered with name. + manifest []byte + exists bool +} + +// checkSandboxConfig returns this build's gVisor SandboxConfig. It fails when +// sandboxConfigExists does. +func (e *Env) checkSandboxConfig(ctx context.Context) (sandboxConfig, error) { + tmpl := e.Cfg.Manifest("sandboxconfig-gvisor.yaml.tmpl") + unnamed, err := render.Template(tmpl, nil, nil) + if err != nil { + return sandboxConfig{}, err + } + name, err := sandboxConfigName(e.SubstrateVersion(), atev1.SandboxClassGvisor, unnamed) if err != nil { - return "", err + return sandboxConfig{}, err } configs, err := e.listSandboxConfigs(ctx) if err != nil { - return "", err + return sandboxConfig{}, err } - // The warning is about the cluster before the apply. - if _, err := checkClassDefault(configs, name, class); err != nil { - return "", err + exists, err := sandboxConfigExists(configs, name) + if err != nil { + return sandboxConfig{}, err } - return name, nil + manifest := render.Expand(string(unnamed), map[string]string{"SANDBOXCONFIG_NAME": name}, nil) + return sandboxConfig{name: name, manifest: manifest, exists: exists}, nil +} + +// sandboxConfigExists reports whether configs hold the SandboxConfig named +// name. It fails when that config is being deleted, because applying it would +// only update the terminating object. +func sandboxConfigExists(configs []atev1.SandboxConfig, name string) (bool, error) { + i := slices.IndexFunc(configs, func(sc atev1.SandboxConfig) bool { return sc.Name == name }) + if i >= 0 && !configs[i].DeletionTimestamp.IsZero() { + return false, fmt.Errorf("SandboxConfig %s is being deleted; wait until it is gone, then run this again", name) + } + return i >= 0, nil } // listSandboxConfigs returns the SandboxConfigs in the cluster. A cluster @@ -133,53 +167,59 @@ func (e *Env) sandboxConfigs() dynamic.ResourceInterface { } // sandboxConfigName returns the name of the SandboxConfig of class that -// Substrate version installs: substrate--, with the version -// lower-cased and every character a name cannot hold replaced by '-'. -func sandboxConfigName(version string, class atev1.SandboxClass) (string, error) { - v := strings.Map(func(r rune) rune { - switch { - case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '-': - return r - case r >= 'A' && r <= 'Z': - return r - 'A' + 'a' +// Substrate version installs. unnamed is the config's manifest rendered +// without its name. +// +// - A release's config is substrate--, with the version +// lower-cased. version is a release when it is a release tag without what +// `git describe` appends to one. +// - Any other build's is substrate-dev--. hash is the first +// 10 hex digits of the SHA-256 of unnamed without its comment lines, so +// builds that render the same config share it. +func sandboxConfigName(version string, class atev1.SandboxClass, unnamed []byte) (string, error) { + var id string + if images.CheckReleaseVersion(version) == nil && !gitDescribeSuffix.MatchString(version) { + id = strings.ToLower(version) + } else { + h := sha256.New() + for _, line := range strings.SplitAfter(string(unnamed), "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "#") { + h.Write([]byte(line)) + } } - return '-' - }, version) - name := fmt.Sprintf("substrate-%s-%s", v, class) + id = "dev-" + hex.EncodeToString(h.Sum(nil))[:10] + } + name := fmt.Sprintf("substrate-%s-%s", id, class) if errs := validation.IsDNS1123Subdomain(name); len(errs) > 0 { return "", fmt.Errorf("version %q gives the invalid SandboxConfig name %q: %s", version, name, strings.Join(errs, "; ")) } return name, nil } -// checkClassDefault checks the SandboxConfig named name, of class. configs -// are the SandboxConfigs in the cluster. +// gitDescribeSuffix matches what `git describe --dirty` appends to a tag: the +// number of commits since it and the commit, and -dirty. A release tag +// allows both as a prerelease. +var gitDescribeSuffix = regexp.MustCompile(`(-[0-9]+-g[0-9a-f]+)(-dirty)?$|-dirty$`) + +// classDefaultWarning returns a warning about the class default of class, or +// "" for none. configs are the SandboxConfigs in the cluster, and name is +// this build's config. // -// - It returns an error when the config is being deleted. Applying it would -// only update the terminating object. // - It warns when the config has the class-default annotation but is not // the class default, because a newer default exists. That happens when an -// older release's config is applied again. +// earlier build's config is deployed again. // - It warns when the class has no default, so pools of the class without a // configRef resolve none. -func checkClassDefault(configs []atev1.SandboxConfig, name string, class atev1.SandboxClass) (string, error) { - var existing *atev1.SandboxConfig - for i := range configs { - if configs[i].Name == name { - existing = &configs[i] - } - } - newest := atev1.ClassDefaultSandboxConfig(configs, class) - if existing != nil && !existing.DeletionTimestamp.IsZero() { - return "", fmt.Errorf("SandboxConfig %s is being deleted; wait until it is gone, then run this again", name) - } - if newest == nil { +func classDefaultWarning(configs []atev1.SandboxConfig, name string, class atev1.SandboxClass) string { + def := atev1.ClassDefaultSandboxConfig(configs, class) + if def == nil { return fmt.Sprintf("no SandboxConfig of sandbox class %q has the %s annotation, so WorkerPools of the class without a configRef resolve no config", - class, atev1.SandboxConfigClassDefaultAnnotation), nil + class, atev1.SandboxConfigClassDefaultAnnotation) } - if existing != nil && existing.Annotations[atev1.SandboxConfigClassDefaultAnnotation] == "true" && newest.Name != name { + i := slices.IndexFunc(configs, func(sc atev1.SandboxConfig) bool { return sc.Name == name }) + if i >= 0 && configs[i].Annotations[atev1.SandboxConfigClassDefaultAnnotation] == "true" && def.Name != name { return fmt.Sprintf("SandboxConfig %s has the %s annotation but is not the %s class default, because %s is newer", - name, atev1.SandboxConfigClassDefaultAnnotation, class, newest.Name), nil + name, atev1.SandboxConfigClassDefaultAnnotation, class, def.Name) } - return "", nil + return "" } diff --git a/cmd/ate-setup/internal/steps/sandboxconfig_test.go b/cmd/ate-setup/internal/steps/sandboxconfig_test.go index e019b9a8b5..509d8cee72 100644 --- a/cmd/ate-setup/internal/steps/sandboxconfig_test.go +++ b/cmd/ate-setup/internal/steps/sandboxconfig_test.go @@ -26,20 +26,25 @@ import ( func TestSandboxConfigName(t *testing.T) { tests := []struct { - name string - version string - want string - wantErr bool + name string + version string + manifest string + want string + wantErr bool }{ - {name: "release tag", version: "v0.4.2", want: "substrate-v0.4.2-gvisor"}, - {name: "git describe of a dirty checkout", version: "v0.4.2-12-gabc1234-dirty", want: "substrate-v0.4.2-12-gabc1234-dirty-gvisor"}, - {name: "upper case and build metadata", version: "V1.0.0+Build", want: "substrate-v1.0.0-build-gvisor"}, - {name: "no git", version: "dev", want: "substrate-dev-gvisor"}, - {name: "dot next to a dash", version: "v1.", wantErr: true}, + {name: "release tag", version: "v0.4.2", manifest: "spec: a\n", want: "substrate-v0.4.2-gvisor"}, + {name: "release candidate in upper case", version: "v1.0.0-RC.1", manifest: "spec: a\n", want: "substrate-v1.0.0-rc.1-gvisor"}, + {name: "git describe of a dirty checkout", version: "v0.4.2-12-gabc1234-dirty", manifest: "spec: a\n", want: "substrate-dev-d5ee99cf6b-gvisor"}, + {name: "comment lines do not count", version: "v0.4.2-12-gabc1234-dirty", manifest: "# a\n # b\nspec: a\n", want: "substrate-dev-d5ee99cf6b-gvisor"}, + {name: "another spec", version: "v0.4.2-12-gabc1234-dirty", manifest: "spec: b\n", want: "substrate-dev-22e34e826a-gvisor"}, + {name: "git describe of a release tag", version: "v0.4.2-3-gabc1234", manifest: "spec: a\n", want: "substrate-dev-d5ee99cf6b-gvisor"}, + {name: "dirty checkout of a release tag", version: "v0.4.2-dirty", manifest: "spec: a\n", want: "substrate-dev-d5ee99cf6b-gvisor"}, + {name: "moving image tag", version: "main", manifest: "spec: a\n", want: "substrate-dev-d5ee99cf6b-gvisor"}, + {name: "release candidate with an empty label", version: "v1.0.0-rc..1", manifest: "spec: a\n", wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := sandboxConfigName(tt.version, atev1.SandboxClassGvisor) + got, err := sandboxConfigName(tt.version, atev1.SandboxClassGvisor, []byte(tt.manifest)) if (err != nil) != tt.wantErr { t.Fatalf("sandboxConfigName(%q) error = %v, wantErr %v", tt.version, err, tt.wantErr) } @@ -50,7 +55,7 @@ func TestSandboxConfigName(t *testing.T) { } } -func TestCheckClassDefault(t *testing.T) { +func TestClassDefaultWarning(t *testing.T) { const name = "substrate-v2-gvisor" t0 := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) config := func(name string, class atev1.SandboxClass, created time.Time, isDefault, deleting bool) atev1.SandboxConfig { @@ -73,7 +78,6 @@ func TestCheckClassDefault(t *testing.T) { configs []atev1.SandboxConfig // wantWarning is a substring of the warning, empty for none. wantWarning string - wantErr bool }{ { name: "fresh cluster", @@ -156,28 +160,47 @@ func TestCheckClassDefault(t *testing.T) { config("substrate-v3-microvm", microvm, t0.Add(time.Hour), true, false), }, }, - { - name: "being deleted, annotated", - configs: []atev1.SandboxConfig{config(name, gvisor, t0, true, true)}, - wantErr: true, - }, - { - name: "being deleted, not annotated", - configs: []atev1.SandboxConfig{ - config("substrate-v1-gvisor", gvisor, t0, true, false), - config(name, gvisor, t0.Add(time.Hour), false, true), - }, - wantErr: true, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - warning, err := checkClassDefault(tt.configs, name, gvisor) + warning := classDefaultWarning(tt.configs, name, gvisor) + if (warning == "") != (tt.wantWarning == "") || !strings.Contains(warning, tt.wantWarning) { + t.Errorf("classDefaultWarning() = %q, want it to contain %q", warning, tt.wantWarning) + } + }) + } +} + +func TestSandboxConfigExists(t *testing.T) { + const name = "substrate-v2-gvisor" + config := func(name string, deleting bool) atev1.SandboxConfig { + sc := atev1.SandboxConfig{ObjectMeta: metav1.ObjectMeta{Name: name}} + if deleting { + sc.DeletionTimestamp = &metav1.Time{Time: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)} + } + return sc + } + + tests := []struct { + name string + configs []atev1.SandboxConfig + wantExists bool + wantErr bool + }{ + {name: "fresh cluster"}, + {name: "only other configs", configs: []atev1.SandboxConfig{config("substrate-v1-gvisor", false)}}, + {name: "exists", configs: []atev1.SandboxConfig{config(name, false)}, wantExists: true}, + {name: "being deleted", configs: []atev1.SandboxConfig{config(name, true)}, wantErr: true}, + {name: "another config being deleted", configs: []atev1.SandboxConfig{config("substrate-v1-gvisor", true)}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + exists, err := sandboxConfigExists(tt.configs, name) if (err != nil) != tt.wantErr { - t.Fatalf("checkClassDefault() error = %v, wantErr %v", err, tt.wantErr) + t.Fatalf("sandboxConfigExists() error = %v, wantErr %v", err, tt.wantErr) } - if (warning == "") != (tt.wantWarning == "") || !strings.Contains(warning, tt.wantWarning) { - t.Errorf("checkClassDefault() warning = %q, want it to contain %q", warning, tt.wantWarning) + if exists != tt.wantExists { + t.Errorf("sandboxConfigExists() = %v, want %v", exists, tt.wantExists) } }) } diff --git a/demos/counter/counter.yaml.tmpl b/demos/counter/counter.yaml.tmpl index cbf77ca037..a2c0b93637 100644 --- a/demos/counter/counter.yaml.tmpl +++ b/demos/counter/counter.yaml.tmpl @@ -40,7 +40,7 @@ spec: workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor # No configRef: the pool follows the class default, the newest SandboxConfig # of the class annotated sandboxconfig.ate.dev/is-class-default=true - # (substrate--gvisor as installed), and moves to a new default when + # (the one ate-setup installs), and moves to a new default when # one appears. sandboxClasses: - name: gvisor diff --git a/docs/api-guide.md b/docs/api-guide.md index f2b68015d6..62f5fbe82e 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -523,7 +523,7 @@ Each entry of `versions` (`SandboxVersionConfig`): | `state` | `string` | Optional. `Enabled` (default) or `Disabled`. The version named by `defaultVersion` cannot be `Disabled`. | | `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A microVM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, is unreachable, or fails with an error it does not recognize, such as GCS refusing its credentials for a public bucket) is then read anonymously, which serves public buckets such as `gs://gvisor`; any other refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (a snapshot location followed by `/atespaces/`), so assets must not be stored there. On GKE, setup-gcp lets the plugin read only the asset bucket, `-assets`, so keep assets there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | -Each Substrate release installs its own gVisor `SandboxConfig`, `substrate--gvisor`, rendered from [`manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl`](../manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl). `ate-setup deploy ate-system` and `ate-setup deploy sandboxconfig` add the class-default annotation to it unless run with `--add-default-annotation=false`, so gVisor `WorkerPool`s use it without any extra setup. If a newer config of the class already has the annotation, the newer one stays the default, and `ate-setup` warns. Configs from earlier releases stay in the cluster. +Each Substrate release installs its own gVisor `SandboxConfig`, `substrate--gvisor`, rendered from [`manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl`](../manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl). A build that is not a release names it `substrate-dev--gvisor`, hashed from the rendered config, so builds with the same config share one. `ate-setup deploy ate-system` and `ate-setup deploy sandboxconfig` add the class-default annotation when they create it, unless run with `--create-with-default-annotation=false`, so gVisor `WorkerPool`s use it without any extra setup. They leave an existing config as it is, annotations included, so use `kubectl annotate` to change its annotation. If a newer config of the class already has the annotation, the newer one stays the default, and `ate-setup` warns. Configs from earlier releases stay in the cluster. ### Class default @@ -560,9 +560,9 @@ metadata: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - defaultVersion: gvisor-6 + defaultVersion: v6 versions: - - name: gvisor-6 + - name: v6 state: Enabled assets: amd64: @@ -579,7 +579,7 @@ spec: A `microvm` `SandboxConfig` supplies the [Kata Containers](https://katacontainers.io/) + [Cloud Hypervisor](https://www.cloudhypervisor.org/) toolchain instead of `runsc`. Each architecture of each version must define the full asset set — `cloud-hypervisor`, `virtiofsd`, `kata-kernel`, and `kata-image` — which a `ValidatingAdmissionPolicy` enforces at apply time. Worker pods for a microVM pool require `/dev/kvm` and nested-virtualization-capable nodes. The controller requests those devices on the pod automatically, and atelet advertises them only where they exist, so placement follows the hardware rather than a node label. Clusters that reserve nested-virt nodes with an `ate.dev/sandboxClass=microvm` taint are still tolerated: advertising a device attracts these pods to capable nodes but repels nothing else from them. The same convention applies to every class: worker pods of a pool tolerate `ate.dev/sandboxClass=:NoSchedule`, so a cluster can reserve a node pool per sandbox class with that taint, and the atelet DaemonSet tolerates the key for any value. On GKE, a pool can also select a ComputeClass with `template.nodeSelector` (`cloud.google.com/compute-class: `): GKE taints the nodes it creates for the class `cloud.google.com/compute-class=:NoSchedule` and adds the matching toleration to the worker pods, and the atelet DaemonSet tolerates that key as well. -`hack/install-microvm-deps.sh --install` stages the assets and creates `substrate--microvm` from [`manifests/microvm/sandboxconfig-microvm.yaml.tmpl`](../manifests/microvm/sandboxconfig-microvm.yaml.tmpl), with the class-default annotation unless run with `--add-default-annotation=false`. See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. +`hack/install-microvm-deps.sh --install` stages the assets and creates `substrate--microvm` from [`manifests/microvm/sandboxconfig-microvm.yaml.tmpl`](../manifests/microvm/sandboxconfig-microvm.yaml.tmpl), or `substrate-dev--microvm` for a build that is not a release, with the class-default annotation unless run with `--create-with-default-annotation=false`. Like the gVisor installers, it leaves an existing config as it is. See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. --- diff --git a/docs/sandboxconfig-upgrade.md b/docs/sandboxconfig-upgrade.md index dea675ceb2..14b734f055 100644 --- a/docs/sandboxconfig-upgrade.md +++ b/docs/sandboxconfig-upgrade.md @@ -24,11 +24,11 @@ If the cluster runs microVM pools, also run this from the new checkout: BUCKET_NAME=$(sed -n 's/^ *bucketName: //p' ~/ate-upgrade/new.yaml) VERSION=$NEW_RELEASE hack/install-microvm-deps.sh --install ``` -The script reads neither `new.yaml` nor the image tag, so the variables give it the bucket and the release. It creates `substrate-$NEW_RELEASE-microvm` the same way as the gVisor config. It first stages the assets at fixed paths in the bucket. If the release changes them, earlier microVM configs no longer match the bucket, and atelet cannot fetch their assets on nodes that have not cached them. MicroVM pools pinned to an earlier config, or left on one by `--add-default-annotation=false`, then need to move to the new config. +The script reads neither `new.yaml` nor the image tag, so the variables give it the bucket and the release. It creates `substrate-$NEW_RELEASE-microvm` the same way as the gVisor config. It first stages the assets at fixed paths in the bucket. If the release changes them, earlier microVM configs no longer match the bucket, and atelet cannot fetch their assets on nodes that have not cached them. MicroVM pools pinned to an earlier config, or left on one by `--create-with-default-annotation=false`, then need to move to the new config. ### Create the configs without moving pools -Add `--add-default-annotation=false` to both commands above. Pass it to `deploy ate-system` in step 4e of the runbook too, or that step adds the annotation and moves the gVisor pools. Later, make each config the class default: +Add `--create-with-default-annotation=false` to both commands above. Later runs leave the configs as they are, including `deploy ate-system` in step 4e of the runbook. Later, make each config the class default: ```bash kubectl annotate sandboxconfig substrate-$NEW_RELEASE-gvisor sandboxconfig.ate.dev/is-class-default=true diff --git a/docs/upgrade.md b/docs/upgrade.md index fafadcc737..e02771f18e 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -232,7 +232,7 @@ To control which pools move and when, and for what happens to suspended actors, go run ./cmd/ate-setup deploy ate-system --config ~/ate-upgrade/new.yaml --setup-csi=none ``` -This applies what the steps above do not, such as the ate-system NetworkPolicies and the bundled PostgreSQL. The components above are already on the new release and do not roll again. `--setup-csi=none` leaves the CSI drivers as they are. Rerunning is safe. If you passed `--add-default-annotation=false` in step 4d, add it here too, or this step annotates `substrate-$NEW_RELEASE-gvisor` and moves the gVisor pools. +This applies what the steps above do not, such as the ate-system NetworkPolicies and the bundled PostgreSQL. The components above are already on the new release and do not roll again. `--setup-csi=none` leaves the CSI drivers as they are. Rerunning is safe. - **Done:** the command exits 0. - **Stuck:** the command fails. [Roll back step 4](#roll-back-step-4). diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index a69021a7ee..e44a9e288e 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -23,13 +23,14 @@ # # On --install: assembles the asset set (assemble.sh; skipped if OUT already # has them), stages the assets under kata-assets/ to the cluster's object -# store bucket (rustfs on kind, GCS on GKE), and creates this release's -# SandboxConfig, substrate--microvm, from the template. The version -# is VERSION, else `git describe`, as for the gVisor config `ate-setup` -# creates. Every asset sha256 is pinned in the template, so the apply only -# substitutes the name and the bucket. With --add-default-annotation (the -# default) the config gets the sandboxconfig.ate.dev/is-class-default -# annotation. +# store bucket (rustfs on kind, GCS on GKE), and creates this build's +# microVM SandboxConfig from the template unless it exists. It is named as +# `ate-setup` names the gVisor config: substrate--microvm for a +# release, else substrate-dev--microvm. Every asset sha256 is pinned in +# the template, so the render only substitutes the name and the bucket. A +# config this run creates gets the sandboxconfig.ate.dev/is-class-default +# annotation, unless --create-with-default-annotation=false. An existing +# config is left as it is, annotations included. # # Nothing here deletes the configs. Like the gVisor configs, they stay until # the SandboxConfig CRD is deleted with the control plane. @@ -48,7 +49,7 @@ # OUT asset dir (default: $PWD/bin/microvm-assets/$ARCH, gitignored). # ATE_INSTALL_KIND "true" for the kind path (stage assets to rustfs); default # false uploads assets to GCS. -# VERSION (optional) Substrate version in the SandboxConfig name +# VERSION (optional) Substrate version that names the SandboxConfig # (default: git describe). set -o errexit -o nounset -o pipefail @@ -68,25 +69,26 @@ ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" usage() { cat <-microvm. - --add-default-annotation With --install, add the class-default annotation - to the config (default: true). + build's microVM SandboxConfig unless it exists. + --create-with-default-annotation + With --install, add the class-default annotation + to the config if this run creates it (default + true). An existing config is left as it is. -h, --help Show this message. EOF } action="" -add_default_annotation=true +create_with_default_annotation=true while [[ $# -gt 0 ]]; do case "$1" in --install) action="install" ;; - --add-default-annotation|--add-default-annotation=true) add_default_annotation=true ;; - --add-default-annotation=false) add_default_annotation=false ;; + --create-with-default-annotation|--create-with-default-annotation=true) create_with_default_annotation=true ;; + --create-with-default-annotation=false) create_with_default_annotation=false ;; -h|--help) usage; exit 0 ;; *) echo "Error: unknown argument $1" >&2; usage; exit 1 ;; esac @@ -148,21 +150,34 @@ microvm_configs="$(get_microvm_configs)" # --- install ---------------------------------------------------------------- -# Name this release's config the way `ate-setup` names the gVisor one when it -# builds from source: VERSION, else `git describe`, lower-cased, with every -# character a name cannot hold replaced by '-'. Checked here so a bad name -# fails before the stage overwrites the bucket's assets. +# Name this build's config the way `ate-setup` names the gVisor one (see +# sandboxConfigName in cmd/ate-setup/internal/steps/sandboxconfig.go). A +# release tag without what `git describe` appends gives +# substrate--microvm. Anything else gives substrate-dev--microvm, +# hashed from the config rendered without its name and comment lines. Checked +# here so a bad name fails before the stage overwrites the bucket's assets. version="${VERSION:-$(git describe --tags --always --dirty 2>/dev/null || true)}" -version="$(tr '[:upper:]' '[:lower:]' <<<"${version:-dev}" | sed 's/[^a-z0-9.-]/-/g')" -SANDBOXCONFIG_NAME="substrate-${version}-microvm" +manifest="$(sed -e "s|\${BUCKET_SCHEME}|${BUCKET_SCHEME}|g" \ + -e "s|\${ASSET_BUCKET_NAME}|${ASSET_BUCKET_NAME}|g" \ + "${MANIFEST_TEMPLATE}")" +# The release tag pattern of CheckReleaseVersion in +# cmd/ate-setup/internal/images/release.go. +if [[ "${version}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ && + ! "${version}" =~ (-[0-9]+-g[0-9a-f]+)(-dirty)?$|-dirty$ ]]; then + config_id="$(tr '[:upper:]' '[:lower:]' <<<"${version}")" +else + config_id="dev-$(sed '/^[[:space:]]*#/d' <<<"${manifest}" | shasum -a 256 | cut -c1-10)" +fi +SANDBOXCONFIG_NAME="substrate-${config_id}-microvm" dns_label='[a-z0-9]([-a-z0-9]*[a-z0-9])?' if [[ ! "${SANDBOXCONFIG_NAME}" =~ ^${dns_label}(\.${dns_label})*$ || ${#SANDBOXCONFIG_NAME} -gt 253 ]]; then echo "Error: version '${version}' gives the invalid SandboxConfig name ${SANDBOXCONFIG_NAME}." >&2 exit 1 fi -# check_configs checks this release's config against the microVM configs in -# $1, as `ate-setup deploy sandboxconfig` checks the gVisor one: +# check_configs checks this build's config against the microVM configs in +# $1, as `ate-setup deploy sandboxconfig` checks the gVisor one. It sets +# exists when the config is in $1. # * it exits when the config is being deleted # * it sets warning when the config has the annotation but a newer config # has it too. The newest one is the class default, so this one is not. @@ -171,9 +186,11 @@ fi check_configs() { local name created is_default deleting local existing_default="" newest_name="" newest_created="" + exists=false while IFS='|' read -r name created is_default deleting; do [[ -n "${name}" ]] || continue if [[ "${name}" == "${SANDBOXCONFIG_NAME}" ]]; then + exists=true existing_default="${is_default}" if [[ -n "${deleting}" ]]; then echo "Error: SandboxConfig ${SANDBOXCONFIG_NAME} is being deleted; wait until it" >&2 @@ -252,21 +269,20 @@ else OUT="${OUT}" BUCKET="${ASSET_BUCKET_NAME}" hack/microvm-assets/stage-to-gcs.sh fi -# --- 3. create this release's microVM SandboxConfig ------------------------- +# --- 3. create this build's microVM SandboxConfig --------------------------- # Every asset is downloaded rather than built, so all four carry committed, # reproducible per-arch shas and the name and the bucket's scheme and name are -# the only substitutions left. -log "Applying SandboxConfig ${SANDBOXCONFIG_NAME} from ${MANIFEST_TEMPLATE}..." -sed -e "s|\${SANDBOXCONFIG_NAME}|${SANDBOXCONFIG_NAME}|g" \ - -e "s|\${BUCKET_SCHEME}|${BUCKET_SCHEME}|g" \ - -e "s|\${ASSET_BUCKET_NAME}|${ASSET_BUCKET_NAME}|g" \ - "${MANIFEST_TEMPLATE}" \ - | run_kubectl apply -f - - -# Annotated outside the template, so a later apply of the template leaves the -# annotation in place. -if [[ "${add_default_annotation}" == "true" ]]; then - run_kubectl annotate --overwrite sandboxconfig "${SANDBOXCONFIG_NAME}" "${DEFAULT_ANNOTATION}=true" +# the only substitutions. An existing config may have been edited since, so it +# is left as it is. +if [[ "${exists}" == "true" ]]; then + log "SandboxConfig ${SANDBOXCONFIG_NAME} exists; leaving it and its annotations as they are." +else + log "Creating SandboxConfig ${SANDBOXCONFIG_NAME} from ${MANIFEST_TEMPLATE}..." + # shellcheck disable=SC2016 # the template's placeholder, not a variable + run_kubectl create -f - <<<"${manifest//'${SANDBOXCONFIG_NAME}'/${SANDBOXCONFIG_NAME}}" + if [[ "${create_with_default_annotation}" == "true" ]]; then + run_kubectl annotate --overwrite sandboxconfig "${SANDBOXCONFIG_NAME}" "${DEFAULT_ANNOTATION}=true" + fi fi # Checked again against the config's real creation time and annotations. microvm_configs="$(get_microvm_configs)" @@ -274,4 +290,4 @@ check_configs "${microvm_configs}" if [[ -n "${warning}" ]]; then echo "Warning: ${warning}" >&2 fi -log "Done. Installed SandboxConfig ${SANDBOXCONFIG_NAME}." +log "Done. SandboxConfig ${SANDBOXCONFIG_NAME} is installed." diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl b/manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl index c82075758c..85652f7bff 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl @@ -14,9 +14,10 @@ # Template for the gVisor (runsc) SandboxConfig each Substrate release # installs. `ate-setup deploy sandboxconfig` and `deploy ate-system` create -# substrate--gvisor from it, and by default annotate it as the class -# default. Editing this file changes only the configs that later releases -# create, and a rerun of the same release updates its own config. +# substrate--gvisor from it, or substrate-dev--gvisor for a +# build that is not a release, and by default annotate it as the class +# default. They leave an existing config as it is, so editing this file +# changes only the configs that later builds create. # # atelet fetches the gVisor release tarball (gvisor.tar.zstd: runsc plus the # gvisor-bin/ helpers runsc requires next to it) matching the worker node's @@ -34,7 +35,7 @@ metadata: name: ${SANDBOXCONFIG_NAME} spec: sandboxClass: gvisor - defaultVersion: gvisor-6 + defaultVersion: v6 versions: # gVisor release 20261026.0. ateworker-gvisor saves container rootfs overlays # (rootfs/) and durable-dir volumes (fs/) as separate checkpoint bundles. That @@ -42,7 +43,7 @@ spec: # ateworker-gvisor also runs the sandbox with no root container # (google/gvisor#15551). This release's restore picks each container's init # process correctly in such a sandbox (google/gvisor@29b97c66d945). - - name: gvisor-6 + - name: v6 state: Enabled assets: amd64: diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index 5c61e5afc7..b9cb7d748c 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -16,9 +16,11 @@ # sandboxconfig-gvisor.yaml.tmpl, which every install applies, this is opt-in: # hack/install-microvm-deps.sh --install stages the asset set (assemble.sh + # stage-to-gcs.sh / stage-to-rustfs.sh), then creates -# substrate--microvm from this template and by default annotates it -# as the class default. Editing this file changes only the configs that later -# releases create. +# substrate--microvm from this template, or +# substrate-dev--microvm for a build that is not a release, and by +# default annotates it as the class default. It leaves an existing config as +# it is, so editing this file changes only the configs that later builds +# create. # # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs) # are FETCHED at runtime from the asset bucket ${ASSET_BUCKET_NAME} under @@ -51,9 +53,9 @@ metadata: name: ${SANDBOXCONFIG_NAME} spec: sandboxClass: microvm - defaultVersion: microvm-2 + defaultVersion: v2 versions: - - name: microvm-2 + - name: v2 state: Enabled assets: arm64: From c70c2be3d4f4fe32f019c27dd41b96043e88253f Mon Sep 17 00:00:00 2001 From: zoezhao Date: Wed, 28 Oct 2026 16:03:24 -0700 Subject: [PATCH 7/9] ate-setup: check only new or changed versions in the SandboxConfig policy Configs of earlier releases now stay in the cluster. The policy checked every version on every update, so once a release tightened it, an older config could not be updated, and atecontroller could not remove its finalizer. On update the policy now checks only versions that are new or whose assets changed, and every version when sandboxClass changes. Rollback deletes the new release's configs instead of removing their class-default annotation, so upgrading again creates them anew. It then restores the old release's policy once the deletes finish. The guide also says when to delete earlier configs, and no longer copies versions into the old config. Signed-off-by: zoezhao --- .../ate/v1/sandboxconfig_validation_test.go | 130 ++++++++++++++++++ docs/api-guide.md | 2 +- docs/sandboxconfig-upgrade.md | 75 ++++++++-- docs/upgrade.md | 8 +- docs/version-skew-policy.md | 4 +- .../ate-install/sandboxconfig-validation.yaml | 16 ++- 6 files changed, 218 insertions(+), 17 deletions(-) diff --git a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go index 03d5d3cb91..7783e6eb8c 100644 --- a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go +++ b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go @@ -137,6 +137,136 @@ func applyVAP(t *testing.T, ctx context.Context) { } } +// removeVAPBinding deletes the shipped policy's binding and waits until the +// apiserver stops enforcing the policy, confirmed by a sentinel that must be +// admitted. +func removeVAPBinding(t *testing.T, ctx context.Context) { + t.Helper() + binding := &unstructured.Unstructured{} + binding.SetAPIVersion("admissionregistration.k8s.io/v1") + binding.SetKind("ValidatingAdmissionPolicyBinding") + binding.SetName("sandboxconfig-assets") + if err := client.IgnoreNotFound(k8sClient.Delete(ctx, binding)); err != nil { + t.Fatalf("delete binding: %v", err) + } + i := 0 + err := wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, 30*time.Second, true, func(ctx context.Context) (bool, error) { + i++ + sc := sandboxConfig(fmt.Sprintf("vap-cooldown-%d", i), SandboxClassGvisor, + map[string]map[string]AssetFile{"amd64": {"other": otherAsset()}}) + if err := k8sClient.Create(ctx, sc); err != nil { + return false, nil + } + return true, k8sClient.Delete(ctx, sc) + }) + if err != nil { + t.Fatalf("VAP did not become inactive: %v", err) + } +} + +// A config that an earlier, looser policy admitted can still be updated, as +// long as the update adds no version and changes no version's assets. +func TestSandboxConfigUpdateUnderStricterPolicy(t *testing.T) { + ctx := t.Context() + runscAssets := func() map[string]map[string]AssetFile { + return map[string]map[string]AssetFile{"amd64": {"runsc": otherAsset()}} + } + // The config before the update. Its versions v1 and v0 have a bare runsc + // asset, which the current policy rejects. + base := func(name string) *SandboxConfig { + sc := withVersions(sandboxConfig(name, SandboxClassGvisor, runscAssets()), sandboxVersion("v0", runscAssets())) + sc.Finalizers = []string{"test.ate.dev/hold"} + sc.Annotations = map[string]string{SandboxConfigClassDefaultAnnotation: "true"} + return sc + } + + tests := []struct { + name string + update func(sc *SandboxConfig) + errMsg string // empty when the update is allowed + }{{ + name: "remove a finalizer", + update: func(sc *SandboxConfig) { sc.Finalizers = nil }, + }, { + name: "remove the class-default annotation", + update: func(sc *SandboxConfig) { sc.Annotations = nil }, + }, { + name: "disable a version", + update: func(sc *SandboxConfig) { sc.Spec.Versions[1].State = ptr.To(SandboxVersionStateDisabled) }, + }, { + name: "delete a version", + update: func(sc *SandboxConfig) { sc.Spec.Versions = sc.Spec.Versions[:1] }, + }, { + name: "change defaultVersion", + update: func(sc *SandboxConfig) { withDefaultVersion(sc, "v0") }, + }, { + name: "add a valid version", + update: func(sc *SandboxConfig) { withVersions(sc, sandboxVersion("v2", gvisorAssets())) }, + }, { + name: "add an invalid version", + update: func(sc *SandboxConfig) { withVersions(sc, sandboxVersion("v2", runscAssets())) }, + errMsg: gvisorAssetMsg, + }, { + name: "change a version's assets", + update: func(sc *SandboxConfig) { + sc.Spec.Versions[1].Assets["arm64"] = map[string]AssetFile{"runsc": otherAsset()} + }, + errMsg: gvisorAssetMsg, + }, { + name: "change sandboxClass", + update: func(sc *SandboxConfig) { sc.Spec.SandboxClass = SandboxClassMicroVM }, + errMsg: "microvm SandboxConfig must define", + }} + + // Create the configs while no policy is enforced, as an earlier release + // might have. + removeVAPBinding(t, ctx) + configs := make([]*SandboxConfig, len(tests)) + for i := range tests { + configs[i] = base(fmt.Sprintf("stricter-policy-%d", i)) + if err := k8sClient.Create(ctx, configs[i]); err != nil { + t.Fatalf("Create() %s: %v", configs[i].Name, err) + } + } + applyVAP(t, ctx) + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sc := configs[i] + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + latest := &SandboxConfig{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(sc), latest); err != nil { + t.Errorf("cleanup: get SandboxConfig %s: %v", sc.Name, err) + return + } + latest.Finalizers = nil + if err := k8sClient.Update(ctx, latest); err != nil { + t.Errorf("cleanup: remove finalizer from SandboxConfig %s: %v", sc.Name, err) + } + if err := client.IgnoreNotFound(k8sClient.Delete(ctx, latest)); err != nil { + t.Errorf("cleanup: delete SandboxConfig %s: %v", sc.Name, err) + } + }) + tt.update(sc) + err := k8sClient.Update(ctx, sc) + if tt.errMsg == "" { + if err != nil { + t.Fatalf("Update() unexpected error: %v", err) + } + return + } + if err == nil { + t.Fatalf("Update() succeeded, want denied") + } + if !strings.Contains(err.Error(), tt.errMsg) { + t.Errorf("Update() error = %q, want it to contain %q", err.Error(), tt.errMsg) + } + }) + } +} + func TestSandboxConfigValidation(t *testing.T) { ctx := t.Context() applyVAP(t, ctx) diff --git a/docs/api-guide.md b/docs/api-guide.md index 62f5fbe82e..7aa7b1ad45 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -521,7 +521,7 @@ Each entry of `versions` (`SandboxVersionConfig`): | :--- | :--- | :--- | | `name` | `string` | **Required.** Identifies the version within the config. A DNS label (lower-case alphanumerics and `-`, at most 63 characters). | | `state` | `string` | Optional. `Enabled` (default) or `Disabled`. The version named by `defaultVersion` cannot be `Disabled`. | -| `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A microVM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, is unreachable, or fails with an error it does not recognize, such as GCS refusing its credentials for a public bucket) is then read anonymously, which serves public buckets such as `gs://gvisor`; any other refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (a snapshot location followed by `/atespaces/`), so assets must not be stored there. On GKE, setup-gcp lets the plugin read only the asset bucket, `-assets`, so keep assets there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | +| `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A microVM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. On update it checks only versions that are new or whose assets changed. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, is unreachable, or fails with an error it does not recognize, such as GCS refusing its credentials for a public bucket) is then read anonymously, which serves public buckets such as `gs://gvisor`; any other refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (a snapshot location followed by `/atespaces/`), so assets must not be stored there. On GKE, setup-gcp lets the plugin read only the asset bucket, `-assets`, so keep assets there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | Each Substrate release installs its own gVisor `SandboxConfig`, `substrate--gvisor`, rendered from [`manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl`](../manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl). A build that is not a release names it `substrate-dev--gvisor`, hashed from the rendered config, so builds with the same config share one. `ate-setup deploy ate-system` and `ate-setup deploy sandboxconfig` add the class-default annotation when they create it, unless run with `--create-with-default-annotation=false`, so gVisor `WorkerPool`s use it without any extra setup. They leave an existing config as it is, annotations included, so use `kubectl annotate` to change its annotation. If a newer config of the class already has the annotation, the newer one stays the default, and `ate-setup` warns. Configs from earlier releases stay in the cluster. diff --git a/docs/sandboxconfig-upgrade.md b/docs/sandboxconfig-upgrade.md index 14b734f055..1c7a3f2d75 100644 --- a/docs/sandboxconfig-upgrade.md +++ b/docs/sandboxconfig-upgrade.md @@ -37,25 +37,82 @@ kubectl annotate sandboxconfig substrate-$NEW_RELEASE-microvm sandboxconfig.ate. ### Check -Each pool's `CONFIG` column shows the config it uses: +Each pool's `IN-USE` column shows the config it uses. `PINNED` is the config its `configRef` names, and `PENDING` is a config it is switching to. ```bash -kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,CLASS:.spec.sandboxClasses[0].name,CONFIG:.status.sandboxClasses[0].configRef.name' +kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,PINNED:.spec.sandboxClasses[*].configRef.name,IN-USE:.status.sandboxClasses[*].configRef.name,PENDING:.status.sandboxClasses[*].pendingConfigRef.name' +``` + +## Delete earlier configs + +Once the upgrade is done, delete the configs of earlier releases that no pool uses, so they do not pile up. Check first that no pool names the config in any column of [Check](#check). A config that a pool uses is not deleted until no pool does. If you roll back later, [Roll back](#roll-back) creates the old release's config again. + +```bash +kubectl delete sandboxconfig ``` ## Roll back -Roll back the SandboxConfigs first in the runbook's [Roll back step 4](upgrade.md#roll-back-step-4), from the old checkout. +Rollback runs from the old checkout, in two parts around the components in the runbook's [Roll back step 4](upgrade.md#roll-back-step-4). + +### Move pools off the new configs + +List the pools as in [Check](#check). Change or remove the `configRef` of each pool whose `PINNED` column names a new release's config. A config is not deleted while a pool uses it, so a pinned pool blocks the delete below. + +List the configs and their class-default annotations: + +```bash +kubectl get sandboxconfigs -o custom-columns='NAME:.metadata.name,CLASS:.spec.sandboxClass,CLASS-DEFAULT:.metadata.annotations.sandboxconfig\.ate\.dev/is-class-default,CREATED:.metadata.creationTimestamp' +``` + +Each class needs a config other than the new release's with `CLASS-DEFAULT` set to `true`, usually the old release's config. Otherwise pools without a `configRef` stay on the new config, and the delete below does not finish. -Actors suspended on a version the old release's config does not list resume without their memory. To keep restoring snapshots taken on a version new in this release, copy that version's entry from the new release's config into the old one before you go on. +If a class has none and the list has the old release's config of that class, annotate that config: + +```bash +kubectl annotate sandboxconfig substrate-$OLD_RELEASE-gvisor sandboxconfig.ate.dev/is-class-default=true +kubectl annotate sandboxconfig substrate-$OLD_RELEASE-microvm sandboxconfig.ate.dev/is-class-default=true +``` + +If the old release's gVisor config is gone, create it. It gets the annotation: + +```bash +go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/old.yaml +``` + +This also applies the old admission policy before the new config is gone. If the delete then does not finish, see [Restore the old admission policy](#restore-the-old-admission-policy). + +If the cluster runs microVM pools, run this. It stages the old release's microVM assets, and creates the old release's microVM config with the annotation if it is gone. + +```bash +BUCKET_NAME=$(sed -n 's/^ *bucketName: //p' ~/ate-upgrade/old.yaml) VERSION=$OLD_RELEASE hack/install-microvm-deps.sh --install +``` + +Then delete the new release's configs: + +```bash +kubectl delete sandboxconfig substrate-$NEW_RELEASE-gvisor substrate-$NEW_RELEASE-microvm --ignore-not-found --wait=false +``` + +A config being deleted is never the class default, so pools without a `configRef` move to the newest other config with the annotation. ate-controller finishes the delete once no pool uses the config. `--wait=false` returns before that, because if the new ate-controller is broken, the delete only finishes after the runbook restores the old one. + +If the old release's config does not list the version an actor was suspended on, the actor resumes without its memory, keeping its files. + +### Restore the old admission policy + +Run this after `deploy ate-controller` in the runbook's [Roll back step 4](upgrade.md#roll-back-step-4). Wait for the new release's configs to be gone, then apply the old release's policy: + +```bash +kubectl wait --for=delete sandboxconfig/substrate-$NEW_RELEASE-gvisor sandboxconfig/substrate-$NEW_RELEASE-microvm --timeout=5m +kubectl apply --server-side --force-conflicts --field-manager=ate-setup -f manifests/ate-install/sandboxconfig-validation.yaml +``` -If the release changed the microVM assets, stage the old ones from the old checkout with `hack/microvm-assets/assemble.sh` and the stage script, as `hack/install-microvm-deps.sh --install` runs them. +ate-controller finishes a delete by removing the config's finalizer, which is an update. The new policy checks only versions that an update adds or whose assets it changes. An older policy may check them, and reject the update if the new config fails its rules. So the wait comes first. -Then remove the class-default annotation from the new release's configs. Skip a line if the upgrade did not create that config. +If the wait times out and a pool still names a new config in [Check](#check), fix that pool. If no pool does, the old policy rejected the update, which can happen after `deploy sandboxconfig` above applied it early. Apply the new release's policy, then run the commands above again: ```bash -kubectl annotate sandboxconfig substrate-$NEW_RELEASE-gvisor sandboxconfig.ate.dev/is-class-default- -kubectl annotate sandboxconfig substrate-$NEW_RELEASE-microvm sandboxconfig.ate.dev/is-class-default- +kubectl apply --server-side --force-conflicts --field-manager=ate-setup -f ~/ate-upgrade/new/manifests/ate-install/sandboxconfig-validation.yaml ``` -The old release's config still has the annotation, so it is the class default again, and pools without a `configRef` move back to it. Pools pinned to a new release's config stay on it until you change their `configRef`. +Apply the policy even if the upgrade created no config, because step 4d applies it first. diff --git a/docs/upgrade.md b/docs/upgrade.md index e02771f18e..575b68bc39 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -224,7 +224,7 @@ To control which pools move and when, and for what happens to suspended actors, - **Done:** the commands exit 0. - **Stuck:** a command fails. [Roll back step 4](#roll-back-step-4). -- **Actors:** a suspended actor whose sandbox version the new config does not list resumes without its memory, keeping its files. +- **Actors:** if the new config does not list the sandbox version an actor was suspended on, the actor resumes without its memory, keeping its files. #### 4e. The rest of the system bundle @@ -240,7 +240,7 @@ This applies what the steps above do not, such as the ate-system NetworkPolicies Then install the new `kubectl ate` with `go install ./cmd/kubectl-ate`. Actor owners can now use API fields and commands new in this release, and you can set WorkerPool, SandboxConfig and CSIDriverConfig fields new in this release. -The upgrade is done when step 4 is done and [Where am I?](#where-am-i) shows every component and pool moved, every pool done rolling and none paused. Keep `~/ate-upgrade` while you might still roll back, then delete it. +The upgrade is done when step 4 is done and [Where am I?](#where-am-i) shows every component and pool moved, every pool done rolling and none paused. Then delete the SandboxConfigs of earlier releases that no pool uses, as in [Delete earlier configs](sandboxconfig-upgrade.md#delete-earlier-configs). Keep `~/ate-upgrade` while you might still roll back, then delete it. ## Something's wrong, or I need to roll back @@ -267,7 +267,7 @@ cd ~/ate-upgrade/old #### Roll back step 4 -Tell actor owners to stop using API fields new in this release, because the old API server rejects them. Roll back the SandboxConfigs as in [Roll back](sandboxconfig-upgrade.md#roll-back) in the SandboxConfig upgrade guide. Then: +Tell actor owners to stop using API fields new in this release, because the old API server rejects them. Follow [Move pools off the new configs](sandboxconfig-upgrade.md#move-pools-off-the-new-configs) in the SandboxConfig upgrade guide. Then: ```bash go run ./cmd/ate-setup deploy atenet --config ~/ate-upgrade/old.yaml @@ -276,6 +276,8 @@ go run ./cmd/ate-setup deploy apiserver --config ~/ate-upgrade/old.yaml go install ./cmd/kubectl-ate ``` +Then follow [Restore the old admission policy](sandboxconfig-upgrade.md#restore-the-old-admission-policy). + `deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. No pool rolls. Fields that the new release added to WorkerPools, SandboxConfigs or CSIDriverConfigs are dropped from those objects. Remove them from your YAML files too, or `kubectl apply` rejects the files. diff --git a/docs/version-skew-policy.md b/docs/version-skew-policy.md index e4c9ae3f23..73418d9463 100644 --- a/docs/version-skew-policy.md +++ b/docs/version-skew-policy.md @@ -82,7 +82,7 @@ Each release creates its own SandboxConfig per sandbox class, such as `substrate A kept version's assets must stay fetchable, so a release does not overwrite them. The microVM assets live at fixed paths in the cluster's bucket, so a microVM release that changes them replaces the version instead. Moving a pool to a new config does not change the pod template, so workers do not roll. -An actor that starts without memory state, new or cold booting, gets the default version. An actor restoring memory state gets a version with the same assets as the one its snapshot was taken on, and stays on it until it next starts without memory state. If the pool's config has no such version, the actor cold boots, keeping its files and losing its memory. Rollback removes the class-default annotation from the N+1 config, so the previous default config before N+1 rollout is the class default again. That config does not list the sandbox versions new in N+1. An actor suspended on one of them resumes without its memory, unless you first add that version to the config. +An actor that starts without memory state, new or cold booting, gets the default version. An actor restoring memory state gets a version with the same assets as the one its snapshot was taken on, and stays on it until it next starts without memory state. If the pool's config has no such version, the actor cold boots, keeping its files and losing its memory. Rollback deletes the N+1 config, so the newest other config with the class-default annotation, usually N's, is the class default again. That config does not list the sandbox versions new in N+1. An actor suspended on one of them resumes without its memory. The [SandboxConfig upgrade guide](sandboxconfig-upgrade.md) covers the details. @@ -124,7 +124,7 @@ To move from N to N+1, in this order, waiting for each step to finish: 8. The rest of the substrate system bundle: what no step above deploys. 9. Clients. -Rollback from N+1 to N is the same list in reverse, with release N, except that the schema migration and step 8 are not undone. Clients stop sending N+1 fields before `ate-api-server` goes back, and `atelet` goes back only after every worker is back at N. Rolling back further than N is not supported. +Rollback from N+1 to N is the same list in reverse, with release N, except that the schema migration and step 8 are not undone. Step 7 deletes the N+1 SandboxConfigs before step 5, and restores the N admission policy after the deletes finish. See [Roll back](sandboxconfig-upgrade.md#roll-back) in the SandboxConfig upgrade guide. Clients stop sending N+1 fields before `ate-api-server` goes back, and `atelet` goes back only after every worker is back at N. Rolling back further than N is not supported. ### Skipping releases diff --git a/manifests/ate-install/sandboxconfig-validation.yaml b/manifests/ate-install/sandboxconfig-validation.yaml index ec28f4457f..ff44f7ddea 100644 --- a/manifests/ate-install/sandboxconfig-validation.yaml +++ b/manifests/ate-install/sandboxconfig-validation.yaml @@ -29,12 +29,24 @@ spec: apiVersions: ["v1"] operations: ["CREATE", "UPDATE"] resources: ["sandboxconfigs"] + # A later release can tighten this policy, and the configs of earlier + # releases stay in the cluster. Check only versions that are new or whose + # assets changed, so those configs can still be updated: removing a + # finalizer, disabling or deleting a version, or changing defaultVersion. + variables: + - name: changedVersions + expression: >- + object.spec.versions.filter(v, + oldObject == null || + oldObject.spec.sandboxClass != object.spec.sandboxClass || + !oldObject.spec.versions.exists(o, o.name == v.name && + has(o.assets) == has(v.assets) && (!has(v.assets) || o.assets == v.assets))) validations: # gVisor needs a "gvisor" release-tarball asset (gvisor.tar.zstd: runsc plus # its gvisor-bin/ helpers) for every architecture it advertises. - expression: >- object.spec.sandboxClass != 'gvisor' || - object.spec.versions.all(v, + variables.changedVersions.all(v, has(v.assets) && size(v.assets) > 0 && v.assets.all(arch, 'gvisor' in v.assets[arch])) message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) asset for every architecture of every version under spec.versions[].assets" @@ -42,7 +54,7 @@ spec: # architecture it advertises. - expression: >- object.spec.sandboxClass != 'microvm' || - object.spec.versions.all(v, + variables.changedVersions.all(v, has(v.assets) && size(v.assets) > 0 && v.assets.all(arch, ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image'] From c631e513b489a55da7f9b0fe2cb81c9600a6e547 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Wed, 28 Oct 2026 17:24:15 -0700 Subject: [PATCH 8/9] ate-setup: name every SandboxConfig by hash A release tag named its config after the version, and only other builds used the hash. Telling the two apart meant parsing git describe output. Every build and release now gets substrate--, so a release that does not change a template reuses the existing config, and pools do not move. The upgrade can no longer predict the new config's name, and it may create none. So a created config is labeled sandboxconfig.ate.dev/created-by-version with the Substrate version, and rollback deletes the configs labeled with the new release. The label is filled in after hashing, so it does not change the name. ate-setup and install-microvm-deps.sh fail early when the version is not a valid label value. Signed-off-by: zoezhao --- .../ate/v1/sandboxconfig_validation_test.go | 2 +- cmd/ate-setup/commands.md | 5 +- cmd/ate-setup/internal/cmd/deploy.go | 8 +-- cmd/ate-setup/internal/images/release.go | 4 +- cmd/ate-setup/internal/steps/benchmarks.go | 4 +- cmd/ate-setup/internal/steps/sandboxconfig.go | 60 +++++++------------ .../internal/steps/sandboxconfig_test.go | 23 ++----- docs/api-guide.md | 12 ++-- docs/sandboxconfig-upgrade.md | 34 ++++++----- docs/upgrade.md | 4 +- docs/version-skew-policy.md | 4 +- hack/install-microvm-deps.sh | 50 +++++++--------- .../sandboxconfig-gvisor.yaml.tmpl | 9 +-- .../microvm/sandboxconfig-microvm.yaml.tmpl | 11 ++-- 14 files changed, 98 insertions(+), 132 deletions(-) diff --git a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go index 7783e6eb8c..676559d9e1 100644 --- a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go +++ b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go @@ -491,7 +491,7 @@ func TestShippedSandboxConfigManifests(t *testing.T) { if err != nil { t.Fatalf("read manifest: %v", err) } - raw = []byte(strings.NewReplacer("${SANDBOXCONFIG_NAME}", "test-"+filepath.Base(path), "${BUCKET_SCHEME}", "s3", "${ASSET_BUCKET_NAME}", "test-bucket").Replace(string(raw))) + raw = []byte(strings.NewReplacer("${SANDBOXCONFIG_NAME}", "test-"+filepath.Base(path), "${SUBSTRATE_VERSION}", "v0.4.2", "${BUCKET_SCHEME}", "s3", "${ASSET_BUCKET_NAME}", "test-bucket").Replace(string(raw))) sc := &SandboxConfig{} if err := yaml.UnmarshalStrict(raw, sc); err != nil { t.Fatalf("decode manifest: %v", err) diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 8288e5bf3f..bb7446439b 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -114,9 +114,8 @@ resource below on the way, so those subcommands are only needed to redo one on a running cluster. `deploy ate-system` and `deploy sandboxconfig` create this build's gVisor -SandboxConfig with the `sandboxconfig.ate.dev/is-class-default` annotation. A -release names it `substrate--gvisor`. Any other build names it -`substrate-dev--gvisor`, hashed from the rendered config. Pass +SandboxConfig with the `sandboxconfig.ate.dev/is-class-default` annotation. It +is named `substrate--gvisor`, hashed from the rendered config. Pass `--create-with-default-annotation=false` to create it without the annotation. An existing config is left as it is, annotations included. diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 0869b9b6f7..d312ab5061 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -151,10 +151,10 @@ var deploySandboxConfigCmd = &cobra.Command{ Use: "sandboxconfig", Short: "Deploy the SandboxConfig admission policy and this build's gVisor SandboxConfig only", Long: `Deploy the SandboxConfig admission policy and create this build's gVisor -SandboxConfig from manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl. A -release names it substrate--gvisor. Any other build names it -substrate-dev--gvisor, hashed from the rendered config, so builds with -the same config share one. +SandboxConfig from manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl. It is +named substrate--gvisor, hashed from the rendered config, so builds with +the same config share one, and labeled +sandboxconfig.ate.dev/created-by-version with the version that creates it. When this run creates the config, it gets the sandboxconfig.ate.dev/is-class-default annotation, unless diff --git a/cmd/ate-setup/internal/images/release.go b/cmd/ate-setup/internal/images/release.go index 7f56370a0a..0510989d09 100644 --- a/cmd/ate-setup/internal/images/release.go +++ b/cmd/ate-setup/internal/images/release.go @@ -48,8 +48,8 @@ const RevisionLabel = "org.opencontainers.image.revision" const VersionLabel = "org.opencontainers.image.version" // releaseTag matches a release tag, vMAJOR.MINOR.PATCH, or a release candidate, -// vMAJOR.MINOR.PATCH-PRERELEASE. hack/release/draft-release.sh and -// hack/install-microvm-deps.sh accept the same. +// vMAJOR.MINOR.PATCH-PRERELEASE. hack/release/draft-release.sh accepts the +// same. var releaseTag = regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$`) // CheckReleaseVersion returns an error unless version is a release tag. diff --git a/cmd/ate-setup/internal/steps/benchmarks.go b/cmd/ate-setup/internal/steps/benchmarks.go index 3388c22c90..b21ac88f2c 100644 --- a/cmd/ate-setup/internal/steps/benchmarks.go +++ b/cmd/ate-setup/internal/steps/benchmarks.go @@ -120,8 +120,8 @@ func (e *Env) DeleteBenchmarks(ctx context.Context, opts BenchmarkOptions) error func (e *Env) runScript(ctx context.Context, relPath string, args ...string) error { cmd := exec.CommandContext(ctx, e.Cfg.Path(relPath), args...) cmd.Dir = e.Cfg.Root - // install-microvm-deps.sh names its SandboxConfig after VERSION. Set it - // so the name matches the gVisor config's, also with prebuilt images. + // install-microvm-deps.sh labels its SandboxConfig with VERSION. Set it + // so the label matches the gVisor config's, also with prebuilt images. cmd.Env = append(e.Cfg.ScriptEnv(), "VERSION="+e.SubstrateVersion()) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/cmd/ate-setup/internal/steps/sandboxconfig.go b/cmd/ate-setup/internal/steps/sandboxconfig.go index 7d7ae5874f..6f50e9973e 100644 --- a/cmd/ate-setup/internal/steps/sandboxconfig.go +++ b/cmd/ate-setup/internal/steps/sandboxconfig.go @@ -20,7 +20,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "regexp" "slices" "strings" @@ -32,7 +31,6 @@ import ( "k8s.io/client-go/dynamic" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" - "go.ate.dev/substrate/cmd/ate-setup/internal/images" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/cmd/ate-setup/internal/log" "go.ate.dev/substrate/cmd/ate-setup/internal/render" @@ -109,18 +107,20 @@ type sandboxConfig struct { exists bool } -// checkSandboxConfig returns this build's gVisor SandboxConfig. It fails when -// sandboxConfigExists does. +// checkSandboxConfig returns this build's gVisor SandboxConfig, labeled with +// the Substrate version that creates it. It fails when the version is not a +// valid label value, or when sandboxConfigExists fails. func (e *Env) checkSandboxConfig(ctx context.Context) (sandboxConfig, error) { + version := e.SubstrateVersion() + if errs := validation.IsValidLabelValue(version); len(errs) > 0 { + return sandboxConfig{}, fmt.Errorf("version %q is not a valid SandboxConfig label value: %s; set VERSION to one", version, strings.Join(errs, "; ")) + } tmpl := e.Cfg.Manifest("sandboxconfig-gvisor.yaml.tmpl") unnamed, err := render.Template(tmpl, nil, nil) if err != nil { return sandboxConfig{}, err } - name, err := sandboxConfigName(e.SubstrateVersion(), atev1.SandboxClassGvisor, unnamed) - if err != nil { - return sandboxConfig{}, err - } + name := sandboxConfigName(atev1.SandboxClassGvisor, unnamed) configs, err := e.listSandboxConfigs(ctx) if err != nil { return sandboxConfig{}, err @@ -129,7 +129,7 @@ func (e *Env) checkSandboxConfig(ctx context.Context) (sandboxConfig, error) { if err != nil { return sandboxConfig{}, err } - manifest := render.Expand(string(unnamed), map[string]string{"SANDBOXCONFIG_NAME": name}, nil) + manifest := render.Expand(string(unnamed), map[string]string{"SANDBOXCONFIG_NAME": name, "SUBSTRATE_VERSION": version}, nil) return sandboxConfig{name: name, manifest: manifest, exists: exists}, nil } @@ -166,41 +166,21 @@ func (e *Env) sandboxConfigs() dynamic.ResourceInterface { return e.Kube.Dynamic.Resource(atev1.GroupVersion.WithResource("sandboxconfigs")) } -// sandboxConfigName returns the name of the SandboxConfig of class that -// Substrate version installs. unnamed is the config's manifest rendered -// without its name. -// -// - A release's config is substrate--, with the version -// lower-cased. version is a release when it is a release tag without what -// `git describe` appends to one. -// - Any other build's is substrate-dev--. hash is the first -// 10 hex digits of the SHA-256 of unnamed without its comment lines, so -// builds that render the same config share it. -func sandboxConfigName(version string, class atev1.SandboxClass, unnamed []byte) (string, error) { - var id string - if images.CheckReleaseVersion(version) == nil && !gitDescribeSuffix.MatchString(version) { - id = strings.ToLower(version) - } else { - h := sha256.New() - for _, line := range strings.SplitAfter(string(unnamed), "\n") { - if !strings.HasPrefix(strings.TrimSpace(line), "#") { - h.Write([]byte(line)) - } +// sandboxConfigName returns the name of the SandboxConfig of class rendered +// as unnamed, the config's manifest without its name and label. The name is +// substrate--. hash is the first 10 hex digits of the SHA-256 of +// unnamed without its comment lines, so builds that render the same config +// share it. Editing the config later does not rename it. +func sandboxConfigName(class atev1.SandboxClass, unnamed []byte) string { + h := sha256.New() + for _, line := range strings.SplitAfter(string(unnamed), "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "#") { + h.Write([]byte(line)) } - id = "dev-" + hex.EncodeToString(h.Sum(nil))[:10] } - name := fmt.Sprintf("substrate-%s-%s", id, class) - if errs := validation.IsDNS1123Subdomain(name); len(errs) > 0 { - return "", fmt.Errorf("version %q gives the invalid SandboxConfig name %q: %s", version, name, strings.Join(errs, "; ")) - } - return name, nil + return fmt.Sprintf("substrate-%s-%s", hex.EncodeToString(h.Sum(nil))[:10], class) } -// gitDescribeSuffix matches what `git describe --dirty` appends to a tag: the -// number of commits since it and the commit, and -dirty. A release tag -// allows both as a prerelease. -var gitDescribeSuffix = regexp.MustCompile(`(-[0-9]+-g[0-9a-f]+)(-dirty)?$|-dirty$`) - // classDefaultWarning returns a warning about the class default of class, or // "" for none. configs are the SandboxConfigs in the cluster, and name is // this build's config. diff --git a/cmd/ate-setup/internal/steps/sandboxconfig_test.go b/cmd/ate-setup/internal/steps/sandboxconfig_test.go index 509d8cee72..a5fd34e4f7 100644 --- a/cmd/ate-setup/internal/steps/sandboxconfig_test.go +++ b/cmd/ate-setup/internal/steps/sandboxconfig_test.go @@ -27,29 +27,18 @@ import ( func TestSandboxConfigName(t *testing.T) { tests := []struct { name string - version string + class atev1.SandboxClass manifest string want string - wantErr bool }{ - {name: "release tag", version: "v0.4.2", manifest: "spec: a\n", want: "substrate-v0.4.2-gvisor"}, - {name: "release candidate in upper case", version: "v1.0.0-RC.1", manifest: "spec: a\n", want: "substrate-v1.0.0-rc.1-gvisor"}, - {name: "git describe of a dirty checkout", version: "v0.4.2-12-gabc1234-dirty", manifest: "spec: a\n", want: "substrate-dev-d5ee99cf6b-gvisor"}, - {name: "comment lines do not count", version: "v0.4.2-12-gabc1234-dirty", manifest: "# a\n # b\nspec: a\n", want: "substrate-dev-d5ee99cf6b-gvisor"}, - {name: "another spec", version: "v0.4.2-12-gabc1234-dirty", manifest: "spec: b\n", want: "substrate-dev-22e34e826a-gvisor"}, - {name: "git describe of a release tag", version: "v0.4.2-3-gabc1234", manifest: "spec: a\n", want: "substrate-dev-d5ee99cf6b-gvisor"}, - {name: "dirty checkout of a release tag", version: "v0.4.2-dirty", manifest: "spec: a\n", want: "substrate-dev-d5ee99cf6b-gvisor"}, - {name: "moving image tag", version: "main", manifest: "spec: a\n", want: "substrate-dev-d5ee99cf6b-gvisor"}, - {name: "release candidate with an empty label", version: "v1.0.0-rc..1", manifest: "spec: a\n", wantErr: true}, + {name: "gvisor", class: atev1.SandboxClassGvisor, manifest: "spec: a\n", want: "substrate-d5ee99cf6b-gvisor"}, + {name: "comment lines do not count", class: atev1.SandboxClassGvisor, manifest: "# a\n # b\nspec: a\n", want: "substrate-d5ee99cf6b-gvisor"}, + {name: "another spec", class: atev1.SandboxClassGvisor, manifest: "spec: b\n", want: "substrate-22e34e826a-gvisor"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := sandboxConfigName(tt.version, atev1.SandboxClassGvisor, []byte(tt.manifest)) - if (err != nil) != tt.wantErr { - t.Fatalf("sandboxConfigName(%q) error = %v, wantErr %v", tt.version, err, tt.wantErr) - } - if got != tt.want { - t.Errorf("sandboxConfigName(%q) = %q, want %q", tt.version, got, tt.want) + if got := sandboxConfigName(tt.class, []byte(tt.manifest)); got != tt.want { + t.Errorf("sandboxConfigName(%q, %q) = %q, want %q", tt.class, tt.manifest, got, tt.want) } }) } diff --git a/docs/api-guide.md b/docs/api-guide.md index 7aa7b1ad45..c17fc9ab6f 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -63,7 +63,7 @@ spec: - name: gvisor # Optional. Pins the pool to one SandboxConfig instead of the class default. # configRef: - # name: substrate-v0.4.2-gvisor + # name: substrate-04e4d62011-gvisor template: labels: project: agent-platform @@ -523,7 +523,7 @@ Each entry of `versions` (`SandboxVersionConfig`): | `state` | `string` | Optional. `Enabled` (default) or `Disabled`. The version named by `defaultVersion` cannot be `Disabled`. | | `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A microVM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. On update it checks only versions that are new or whose assets changed. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, is unreachable, or fails with an error it does not recognize, such as GCS refusing its credentials for a public bucket) is then read anonymously, which serves public buckets such as `gs://gvisor`; any other refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (a snapshot location followed by `/atespaces/`), so assets must not be stored there. On GKE, setup-gcp lets the plugin read only the asset bucket, `-assets`, so keep assets there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | -Each Substrate release installs its own gVisor `SandboxConfig`, `substrate--gvisor`, rendered from [`manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl`](../manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl). A build that is not a release names it `substrate-dev--gvisor`, hashed from the rendered config, so builds with the same config share one. `ate-setup deploy ate-system` and `ate-setup deploy sandboxconfig` add the class-default annotation when they create it, unless run with `--create-with-default-annotation=false`, so gVisor `WorkerPool`s use it without any extra setup. They leave an existing config as it is, annotations included, so use `kubectl annotate` to change its annotation. If a newer config of the class already has the annotation, the newer one stays the default, and `ate-setup` warns. Configs from earlier releases stay in the cluster. +Substrate installs a gVisor `SandboxConfig` rendered from [`manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl`](../manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl). It is named `substrate--gvisor`, hashed from the rendered config, so builds and releases with the same config share one. Editing the config does not rename it. The label `sandboxconfig.ate.dev/created-by-version` records the Substrate version that created it. `ate-setup deploy ate-system` and `ate-setup deploy sandboxconfig` add the class-default annotation when they create it, unless run with `--create-with-default-annotation=false`, so gVisor `WorkerPool`s use it without any extra setup. They leave an existing config as it is, annotations included, so use `kubectl annotate` to change its annotation. If a newer config of the class already has the annotation, the newer one stays the default, and `ate-setup` warns. Configs from earlier builds stay in the cluster. ### Class default @@ -545,7 +545,7 @@ atecontroller keeps a `SandboxConfig` from being deleted while a `WorkerPool` us A delete cannot be undone, and an object that is being deleted cannot be recreated under the same name. Applying its manifest again only updates the terminating object. To recover from deleting a class default that pools still use: 1. Create a replacement under a different name with the `is-class-default` annotation. Pools move to it on their own, the finalizer is released, and the old config disappears. -2. Once the old config is gone, `ate-setup deploy sandboxconfig` recreates the release's config under its original name. Pools move back to it, because it is then the newest default. +2. Once the old config is gone, `ate-setup deploy sandboxconfig` recreates this build's config under its original name. Pools move back to it, because it is then the newest default. atecontroller alone adds and releases the finalizer. While it is down, `kubectl delete sandboxconfig` of a config that carries the finalizer hangs until atecontroller is back. `ate-setup` strips the finalizer from every config when it deletes the control plane. @@ -555,7 +555,9 @@ atecontroller alone adds and releases the finalizer. While it is down, `kubectl apiVersion: ate.dev/v1 kind: SandboxConfig metadata: - name: substrate-v0.4.2-gvisor + name: substrate-04e4d62011-gvisor + labels: + sandboxconfig.ate.dev/created-by-version: "v0.4.2" annotations: sandboxconfig.ate.dev/is-class-default: "true" spec: @@ -579,7 +581,7 @@ spec: A `microvm` `SandboxConfig` supplies the [Kata Containers](https://katacontainers.io/) + [Cloud Hypervisor](https://www.cloudhypervisor.org/) toolchain instead of `runsc`. Each architecture of each version must define the full asset set — `cloud-hypervisor`, `virtiofsd`, `kata-kernel`, and `kata-image` — which a `ValidatingAdmissionPolicy` enforces at apply time. Worker pods for a microVM pool require `/dev/kvm` and nested-virtualization-capable nodes. The controller requests those devices on the pod automatically, and atelet advertises them only where they exist, so placement follows the hardware rather than a node label. Clusters that reserve nested-virt nodes with an `ate.dev/sandboxClass=microvm` taint are still tolerated: advertising a device attracts these pods to capable nodes but repels nothing else from them. The same convention applies to every class: worker pods of a pool tolerate `ate.dev/sandboxClass=:NoSchedule`, so a cluster can reserve a node pool per sandbox class with that taint, and the atelet DaemonSet tolerates the key for any value. On GKE, a pool can also select a ComputeClass with `template.nodeSelector` (`cloud.google.com/compute-class: `): GKE taints the nodes it creates for the class `cloud.google.com/compute-class=:NoSchedule` and adds the matching toleration to the worker pods, and the atelet DaemonSet tolerates that key as well. -`hack/install-microvm-deps.sh --install` stages the assets and creates `substrate--microvm` from [`manifests/microvm/sandboxconfig-microvm.yaml.tmpl`](../manifests/microvm/sandboxconfig-microvm.yaml.tmpl), or `substrate-dev--microvm` for a build that is not a release, with the class-default annotation unless run with `--create-with-default-annotation=false`. Like the gVisor installers, it leaves an existing config as it is. See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. +`hack/install-microvm-deps.sh --install` stages the assets and creates `substrate--microvm` from [`manifests/microvm/sandboxconfig-microvm.yaml.tmpl`](../manifests/microvm/sandboxconfig-microvm.yaml.tmpl), hashed from the rendered config and labeled with `VERSION`, with the class-default annotation unless run with `--create-with-default-annotation=false`. Like the gVisor installers, it leaves an existing config as it is. See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. --- diff --git a/docs/sandboxconfig-upgrade.md b/docs/sandboxconfig-upgrade.md index 1c7a3f2d75..d79b62f4da 100644 --- a/docs/sandboxconfig-upgrade.md +++ b/docs/sandboxconfig-upgrade.md @@ -2,7 +2,7 @@ This guide covers the SandboxConfig part of an upgrade: step 4d of the [upgrade runbook](upgrade.md), and its rollback. Run it from the runbook's shell, with `NEW_RELEASE` set and `~/ate-upgrade` in place. -Each release creates its own SandboxConfig per sandbox class, `substrate--gvisor` and `substrate--microvm`. Configs from earlier releases stay in the cluster. A WorkerPool without a `configRef` uses the newest config of its class with the `sandboxconfig.ate.dev/is-class-default` annotation. A pool with a `configRef` uses that config. [Class default](api-guide.md#class-default) in the API guide has the full rules. Moving a pool to another config does not roll its workers. +Each sandbox class has a SandboxConfig rendered from a template in the repository, named `substrate--gvisor` or `substrate--microvm` after a hash of the rendered config. A release that changes a template creates a new config, labeled `sandboxconfig.ate.dev/created-by-version=`. A release that does not reuses the existing one. Configs from earlier releases stay in the cluster. A WorkerPool without a `configRef` uses the newest config of its class with the `sandboxconfig.ate.dev/is-class-default` annotation. A pool with a `configRef` uses that config. [Class default](api-guide.md#class-default) in the API guide has the full rules. Moving a pool to another config does not roll its workers. New actors start on the default version of their pool's config. A suspended actor restores its memory only on a version with the same assets as the one its snapshot was taken on. If the pool's config has no such version, the actor resumes without its memory, keeping its files. The [version skew policy](version-skew-policy.md#sandboxconfig) says which versions a release keeps. @@ -14,7 +14,7 @@ New actors start on the default version of their pool's config. A suspended acto go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml ``` -This applies the SandboxConfig admission policy and creates `substrate-$NEW_RELEASE-gvisor` as the gVisor class default. gVisor pools without a `configRef` move to it. Pools pinned to a config with `configRef` stay on it until you change their `configRef`. +This applies the SandboxConfig admission policy and creates the new release's gVisor config as the class default, unless a config with the same hash exists. It prints the config's name. gVisor pools without a `configRef` move to it. Pools pinned to a config with `configRef` stay on it until you change their `configRef`. ### microVM @@ -24,15 +24,14 @@ If the cluster runs microVM pools, also run this from the new checkout: BUCKET_NAME=$(sed -n 's/^ *bucketName: //p' ~/ate-upgrade/new.yaml) VERSION=$NEW_RELEASE hack/install-microvm-deps.sh --install ``` -The script reads neither `new.yaml` nor the image tag, so the variables give it the bucket and the release. It creates `substrate-$NEW_RELEASE-microvm` the same way as the gVisor config. It first stages the assets at fixed paths in the bucket. If the release changes them, earlier microVM configs no longer match the bucket, and atelet cannot fetch their assets on nodes that have not cached them. MicroVM pools pinned to an earlier config, or left on one by `--create-with-default-annotation=false`, then need to move to the new config. +The script reads neither `new.yaml` nor the image tag, so the variables give it the bucket and the release for the label. It creates the new release's microVM config the same way as the gVisor config. It first stages the assets at fixed paths in the bucket. If the release changes them, earlier microVM configs no longer match the bucket, and atelet cannot fetch their assets on nodes that have not cached them. MicroVM pools pinned to an earlier config, or left on one by `--create-with-default-annotation=false`, then need to move to the new config. ### Create the configs without moving pools -Add `--create-with-default-annotation=false` to both commands above. Later runs leave the configs as they are, including `deploy ate-system` in step 4e of the runbook. Later, make each config the class default: +Add `--create-with-default-annotation=false` to both commands above. Later runs leave the configs as they are, including `deploy ate-system` in step 4e of the runbook. Later, annotate each config they created, using the name they printed: ```bash -kubectl annotate sandboxconfig substrate-$NEW_RELEASE-gvisor sandboxconfig.ate.dev/is-class-default=true -kubectl annotate sandboxconfig substrate-$NEW_RELEASE-microvm sandboxconfig.ate.dev/is-class-default=true +kubectl annotate sandboxconfig sandboxconfig.ate.dev/is-class-default=true ``` ### Check @@ -45,7 +44,7 @@ kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME ## Delete earlier configs -Once the upgrade is done, delete the configs of earlier releases that no pool uses, so they do not pile up. Check first that no pool names the config in any column of [Check](#check). A config that a pool uses is not deleted until no pool does. If you roll back later, [Roll back](#roll-back) creates the old release's config again. +Once the upgrade is done, delete the earlier configs that no pool uses, so they do not pile up. Check first that no pool names the config in any column of [Check](#check). A config that a pool uses is not deleted until no pool does. If you roll back later, [Roll back](#roll-back) creates the old release's config again. ```bash kubectl delete sandboxconfig @@ -57,7 +56,13 @@ Rollback runs from the old checkout, in two parts around the components in the r ### Move pools off the new configs -List the pools as in [Check](#check). Change or remove the `configRef` of each pool whose `PINNED` column names a new release's config. A config is not deleted while a pool uses it, so a pinned pool blocks the delete below. +The new configs are the ones the upgrade created, labeled with `NEW_RELEASE`. If the release did not change a template, there are none. List them: + +```bash +kubectl get sandboxconfigs -l sandboxconfig.ate.dev/created-by-version=$NEW_RELEASE +``` + +List the pools as in [Check](#check). Change or remove the `configRef` of each pool whose `PINNED` column names a new config. A config is not deleted while a pool uses it, so a pinned pool blocks the delete below. List the configs and their class-default annotations: @@ -65,13 +70,12 @@ List the configs and their class-default annotations: kubectl get sandboxconfigs -o custom-columns='NAME:.metadata.name,CLASS:.spec.sandboxClass,CLASS-DEFAULT:.metadata.annotations.sandboxconfig\.ate\.dev/is-class-default,CREATED:.metadata.creationTimestamp' ``` -Each class needs a config other than the new release's with `CLASS-DEFAULT` set to `true`, usually the old release's config. Otherwise pools without a `configRef` stay on the new config, and the delete below does not finish. +Each class needs a config outside the new ones with `CLASS-DEFAULT` set to `true`, usually the old release's config. Otherwise pools without a `configRef` stay on the new config, and the delete below does not finish. If a class has none and the list has the old release's config of that class, annotate that config: ```bash -kubectl annotate sandboxconfig substrate-$OLD_RELEASE-gvisor sandboxconfig.ate.dev/is-class-default=true -kubectl annotate sandboxconfig substrate-$OLD_RELEASE-microvm sandboxconfig.ate.dev/is-class-default=true +kubectl annotate sandboxconfig sandboxconfig.ate.dev/is-class-default=true ``` If the old release's gVisor config is gone, create it. It gets the annotation: @@ -88,10 +92,10 @@ If the cluster runs microVM pools, run this. It stages the old release's microVM BUCKET_NAME=$(sed -n 's/^ *bucketName: //p' ~/ate-upgrade/old.yaml) VERSION=$OLD_RELEASE hack/install-microvm-deps.sh --install ``` -Then delete the new release's configs: +Then delete the new configs: ```bash -kubectl delete sandboxconfig substrate-$NEW_RELEASE-gvisor substrate-$NEW_RELEASE-microvm --ignore-not-found --wait=false +kubectl delete sandboxconfigs -l sandboxconfig.ate.dev/created-by-version=$NEW_RELEASE --wait=false ``` A config being deleted is never the class default, so pools without a `configRef` move to the newest other config with the annotation. ate-controller finishes the delete once no pool uses the config. `--wait=false` returns before that, because if the new ate-controller is broken, the delete only finishes after the runbook restores the old one. @@ -100,10 +104,10 @@ If the old release's config does not list the version an actor was suspended on, ### Restore the old admission policy -Run this after `deploy ate-controller` in the runbook's [Roll back step 4](upgrade.md#roll-back-step-4). Wait for the new release's configs to be gone, then apply the old release's policy: +Run this after `deploy ate-controller` in the runbook's [Roll back step 4](upgrade.md#roll-back-step-4). Wait for the new configs to be gone, then apply the old release's policy. ```bash -kubectl wait --for=delete sandboxconfig/substrate-$NEW_RELEASE-gvisor sandboxconfig/substrate-$NEW_RELEASE-microvm --timeout=5m +kubectl wait --for=delete sandboxconfigs -l sandboxconfig.ate.dev/created-by-version=$NEW_RELEASE --timeout=5m kubectl apply --server-side --force-conflicts --field-manager=ate-setup -f manifests/ate-install/sandboxconfig-validation.yaml ``` diff --git a/docs/upgrade.md b/docs/upgrade.md index 575b68bc39..8be0536b1b 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -218,7 +218,7 @@ kubectl -n ate-system rollout status deploy/atenet-egress go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml ``` -By default this creates this release's gVisor SandboxConfig, `substrate-$NEW_RELEASE-gvisor`, as the gVisor class default. gVisor pools without a `configRef` move to it without rolling their workers. If the cluster runs microVM pools, also follow [microVM](sandboxconfig-upgrade.md#microvm) in the SandboxConfig upgrade guide. +By default this creates the new release's gVisor SandboxConfig as the gVisor class default, and gVisor pools without a `configRef` move to it without rolling their workers. If the release did not change the config, it exists already and nothing changes. If the cluster runs microVM pools, also follow [microVM](sandboxconfig-upgrade.md#microvm) in the SandboxConfig upgrade guide. To control which pools move and when, and for what happens to suspended actors, see the [SandboxConfig upgrade guide](sandboxconfig-upgrade.md). @@ -240,7 +240,7 @@ This applies what the steps above do not, such as the ate-system NetworkPolicies Then install the new `kubectl ate` with `go install ./cmd/kubectl-ate`. Actor owners can now use API fields and commands new in this release, and you can set WorkerPool, SandboxConfig and CSIDriverConfig fields new in this release. -The upgrade is done when step 4 is done and [Where am I?](#where-am-i) shows every component and pool moved, every pool done rolling and none paused. Then delete the SandboxConfigs of earlier releases that no pool uses, as in [Delete earlier configs](sandboxconfig-upgrade.md#delete-earlier-configs). Keep `~/ate-upgrade` while you might still roll back, then delete it. +The upgrade is done when step 4 is done and [Where am I?](#where-am-i) shows every component and pool moved, every pool done rolling and none paused. Then delete the earlier SandboxConfigs that no pool uses, as in [Delete earlier configs](sandboxconfig-upgrade.md#delete-earlier-configs). Keep `~/ate-upgrade` while you might still roll back, then delete it. ## Something's wrong, or I need to roll back diff --git a/docs/version-skew-policy.md b/docs/version-skew-policy.md index 73418d9463..68e7798f2a 100644 --- a/docs/version-skew-policy.md +++ b/docs/version-skew-policy.md @@ -78,11 +78,11 @@ Between compatible releases, a CRD change only adds optional fields. The CRDs se ### SandboxConfig -Each release creates its own SandboxConfig per sandbox class, such as `substrate-v0.4.2-gvisor`, from a template in the repository. The config lists the sandbox versions a cluster may run. By default the upgrade annotates it as the class default, and pools without a `configRef` move to it. The previous release's config stays in the cluster. A release that changes a sandbox version adds the new one to the template, points `defaultVersion` at it, and keeps the previous release's version unless that version has a known vulnerability. +Each sandbox class has a SandboxConfig rendered from a template in the repository and named after a hash of the rendered config, such as `substrate-04e4d62011-gvisor`. The config lists the sandbox versions a cluster may run. A release that changes the template creates a new config. By default the upgrade annotates it as the class default, and pools without a `configRef` move to it. The previous release's config stays in the cluster. A release that changes a sandbox version adds the new one to the template, points `defaultVersion` at it, and keeps the previous release's version unless that version has a known vulnerability. A kept version's assets must stay fetchable, so a release does not overwrite them. The microVM assets live at fixed paths in the cluster's bucket, so a microVM release that changes them replaces the version instead. Moving a pool to a new config does not change the pod template, so workers do not roll. -An actor that starts without memory state, new or cold booting, gets the default version. An actor restoring memory state gets a version with the same assets as the one its snapshot was taken on, and stays on it until it next starts without memory state. If the pool's config has no such version, the actor cold boots, keeping its files and losing its memory. Rollback deletes the N+1 config, so the newest other config with the class-default annotation, usually N's, is the class default again. That config does not list the sandbox versions new in N+1. An actor suspended on one of them resumes without its memory. +An actor that starts without memory state, new or cold booting, gets the default version. An actor restoring memory state gets a version with the same assets as the one its snapshot was taken on, and stays on it until it next starts without memory state. If the pool's config has no such version, the actor cold boots, keeping its files and losing its memory. Rollback deletes the configs N+1 created, if any, so the newest other config with the class-default annotation, usually N's, is the class default again. That config does not list the sandbox versions new in N+1. An actor suspended on one of them resumes without its memory. The [SandboxConfig upgrade guide](sandboxconfig-upgrade.md) covers the details. diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index e44a9e288e..ae193f25c5 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -25,12 +25,12 @@ # has them), stages the assets under kata-assets/ to the cluster's object # store bucket (rustfs on kind, GCS on GKE), and creates this build's # microVM SandboxConfig from the template unless it exists. It is named as -# `ate-setup` names the gVisor config: substrate--microvm for a -# release, else substrate-dev--microvm. Every asset sha256 is pinned in -# the template, so the render only substitutes the name and the bucket. A -# config this run creates gets the sandboxconfig.ate.dev/is-class-default -# annotation, unless --create-with-default-annotation=false. An existing -# config is left as it is, annotations included. +# `ate-setup` names the gVisor config, substrate--microvm, and labeled +# with VERSION. Every asset sha256 is pinned in the template, so the render +# only substitutes the name, the label and the bucket. A config this run +# creates gets the sandboxconfig.ate.dev/is-class-default annotation, unless +# --create-with-default-annotation=false. An existing config is left as it +# is, annotations included. # # Nothing here deletes the configs. Like the gVisor configs, they stay until # the SandboxConfig CRD is deleted with the control plane. @@ -49,8 +49,8 @@ # OUT asset dir (default: $PWD/bin/microvm-assets/$ARCH, gitignored). # ATE_INSTALL_KIND "true" for the kind path (stage assets to rustfs); default # false uploads assets to GCS. -# VERSION (optional) Substrate version that names the SandboxConfig -# (default: git describe). +# VERSION (optional) Substrate version the SandboxConfig is labeled +# with (default: git describe). set -o errexit -o nounset -o pipefail @@ -151,27 +151,17 @@ microvm_configs="$(get_microvm_configs)" # --- install ---------------------------------------------------------------- # Name this build's config the way `ate-setup` names the gVisor one (see -# sandboxConfigName in cmd/ate-setup/internal/steps/sandboxconfig.go). A -# release tag without what `git describe` appends gives -# substrate--microvm. Anything else gives substrate-dev--microvm, -# hashed from the config rendered without its name and comment lines. Checked -# here so a bad name fails before the stage overwrites the bucket's assets. -version="${VERSION:-$(git describe --tags --always --dirty 2>/dev/null || true)}" +# sandboxConfigName in cmd/ate-setup/internal/steps/sandboxconfig.go): +# substrate--microvm, hashed from the config rendered without its name, +# label and comment lines. The version is checked here so a bad label value +# fails before the stage overwrites the bucket's assets. manifest="$(sed -e "s|\${BUCKET_SCHEME}|${BUCKET_SCHEME}|g" \ -e "s|\${ASSET_BUCKET_NAME}|${ASSET_BUCKET_NAME}|g" \ "${MANIFEST_TEMPLATE}")" -# The release tag pattern of CheckReleaseVersion in -# cmd/ate-setup/internal/images/release.go. -if [[ "${version}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ && - ! "${version}" =~ (-[0-9]+-g[0-9a-f]+)(-dirty)?$|-dirty$ ]]; then - config_id="$(tr '[:upper:]' '[:lower:]' <<<"${version}")" -else - config_id="dev-$(sed '/^[[:space:]]*#/d' <<<"${manifest}" | shasum -a 256 | cut -c1-10)" -fi -SANDBOXCONFIG_NAME="substrate-${config_id}-microvm" -dns_label='[a-z0-9]([-a-z0-9]*[a-z0-9])?' -if [[ ! "${SANDBOXCONFIG_NAME}" =~ ^${dns_label}(\.${dns_label})*$ || ${#SANDBOXCONFIG_NAME} -gt 253 ]]; then - echo "Error: version '${version}' gives the invalid SandboxConfig name ${SANDBOXCONFIG_NAME}." >&2 +SANDBOXCONFIG_NAME="substrate-$(sed '/^[[:space:]]*#/d' <<<"${manifest}" | shasum -a 256 | cut -c1-10)-microvm" +version="${VERSION:-$(git describe --tags --always --dirty 2>/dev/null || true)}" +if [[ ! "${version}" =~ ^([A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?)?$ || ${#version} -gt 63 ]]; then + echo "Error: version '${version}' is not a valid SandboxConfig label value; set VERSION to one." >&2 exit 1 fi @@ -271,15 +261,15 @@ fi # --- 3. create this build's microVM SandboxConfig --------------------------- # Every asset is downloaded rather than built, so all four carry committed, -# reproducible per-arch shas and the name and the bucket's scheme and name are -# the only substitutions. An existing config may have been edited since, so it +# reproducible per-arch shas and the name, the label, and the bucket's scheme +# and name are the only substitutions. An existing config may have been edited since, so it # is left as it is. if [[ "${exists}" == "true" ]]; then log "SandboxConfig ${SANDBOXCONFIG_NAME} exists; leaving it and its annotations as they are." else log "Creating SandboxConfig ${SANDBOXCONFIG_NAME} from ${MANIFEST_TEMPLATE}..." - # shellcheck disable=SC2016 # the template's placeholder, not a variable - run_kubectl create -f - <<<"${manifest//'${SANDBOXCONFIG_NAME}'/${SANDBOXCONFIG_NAME}}" + run_kubectl create -f - <<<"$(sed -e "s|\${SANDBOXCONFIG_NAME}|${SANDBOXCONFIG_NAME}|g" \ + -e "s|\${SUBSTRATE_VERSION}|${version}|g" <<<"${manifest}")" if [[ "${create_with_default_annotation}" == "true" ]]; then run_kubectl annotate --overwrite sandboxconfig "${SANDBOXCONFIG_NAME}" "${DEFAULT_ANNOTATION}=true" fi diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl b/manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl index 85652f7bff..800fedad68 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml.tmpl @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Template for the gVisor (runsc) SandboxConfig each Substrate release -# installs. `ate-setup deploy sandboxconfig` and `deploy ate-system` create -# substrate--gvisor from it, or substrate-dev--gvisor for a -# build that is not a release, and by default annotate it as the class +# Template for the gVisor (runsc) SandboxConfig. `ate-setup deploy +# sandboxconfig` and `deploy ate-system` create substrate--gvisor from +# it, hashed from the rendered config, and by default annotate it as the class # default. They leave an existing config as it is, so editing this file # changes only the configs that later builds create. # @@ -33,6 +32,8 @@ apiVersion: ate.dev/v1 kind: SandboxConfig metadata: name: ${SANDBOXCONFIG_NAME} + labels: + sandboxconfig.ate.dev/created-by-version: "${SUBSTRATE_VERSION}" spec: sandboxClass: gvisor defaultVersion: v6 diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index b9cb7d748c..9075c344d9 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -16,11 +16,10 @@ # sandboxconfig-gvisor.yaml.tmpl, which every install applies, this is opt-in: # hack/install-microvm-deps.sh --install stages the asset set (assemble.sh + # stage-to-gcs.sh / stage-to-rustfs.sh), then creates -# substrate--microvm from this template, or -# substrate-dev--microvm for a build that is not a release, and by -# default annotates it as the class default. It leaves an existing config as -# it is, so editing this file changes only the configs that later builds -# create. +# substrate--microvm from this template, hashed from the rendered +# config, and by default annotates it as the class default. It leaves an +# existing config as it is, so editing this file changes only the configs +# that later builds create. # # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs) # are FETCHED at runtime from the asset bucket ${ASSET_BUCKET_NAME} under @@ -51,6 +50,8 @@ apiVersion: ate.dev/v1 kind: SandboxConfig metadata: name: ${SANDBOXCONFIG_NAME} + labels: + sandboxconfig.ate.dev/created-by-version: "${SUBSTRATE_VERSION}" spec: sandboxClass: microvm defaultVersion: v2 From f41a4b5fe38c987a25982820f5762fcf4c1bf2a0 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Wed, 28 Oct 2026 19:06:12 -0700 Subject: [PATCH 9/9] ate-setup: create the SandboxConfig with its class-default annotation The installers created the config, then annotated it in a second request. If the second request failed, the config stayed without the annotation, and a rerun found it and left it that way. ate-setup and install-microvm-deps.sh now put the annotation into the created object. ate-setup creates the config with Create instead of server-side apply, so a config created by someone else after the existence check is not overwritten. Kube.CreateOne retries the same failures as an apply and reports AlreadyExists as an existing object. Signed-off-by: zoezhao --- cmd/ate-setup/internal/kube/apply.go | 39 ++++++-- cmd/ate-setup/internal/kube/apply_test.go | 59 +++++++++++- cmd/ate-setup/internal/steps/sandboxconfig.go | 90 +++++++++---------- .../internal/steps/sandboxconfig_test.go | 18 ++-- hack/install-microvm-deps.sh | 7 +- 5 files changed, 147 insertions(+), 66 deletions(-) diff --git a/cmd/ate-setup/internal/kube/apply.go b/cmd/ate-setup/internal/kube/apply.go index 9d92dbb08f..e1ee74b65f 100644 --- a/cmd/ate-setup/internal/kube/apply.go +++ b/cmd/ate-setup/internal/kube/apply.go @@ -65,11 +65,11 @@ func (c *Client) Apply(ctx context.Context, objs []*unstructured.Unstructured) e return nil } -// applyBackoff paces the retries of a failed apply. Five attempts over roughly -// eight seconds covers the two things that routinely fail here — a validating -// webhook whose pod is still starting, and a namespace or CRD whose creation -// has not reached the apiserver handling this request — without turning a -// genuinely broken cluster into a long wait. +// applyBackoff paces the retries of a failed apply or create. Five attempts +// over roughly eight seconds covers the two things that routinely fail here — +// a validating webhook whose pod is still starting, and a namespace or CRD +// whose creation has not reached the apiserver handling this request — without +// turning a genuinely broken cluster into a long wait. var applyBackoff = wait.Backoff{ Steps: 5, Duration: 500 * time.Millisecond, @@ -111,6 +111,34 @@ func applyWithRetry(ctx context.Context, ri dynamicResource, obj *unstructured.U return nil } +// CreateOne creates obj unless an object of its name exists, retrying the +// failures that clear on their own. It reports whether it created obj. +func (c *Client) CreateOne(ctx context.Context, obj *unstructured.Unstructured) (bool, error) { + ri, err := c.resourceFor(obj) + if err != nil { + return false, err + } + return createWithRetry(ctx, ri, obj) +} + +func createWithRetry(ctx context.Context, ri dynamicResource, obj *unstructured.Unstructured) (bool, error) { + retryable := func(err error) bool { + return ctx.Err() == nil && retryableWaitError(err) + } + err := retry.OnError(applyBackoff, retryable, func() error { + _, err := ri.Create(ctx, obj, metav1.CreateOptions{FieldManager: FieldManager}) + return err + }) + // A retry after a lost response can find the object the first attempt + // created, and so reports false. + if apierrors.IsAlreadyExists(err) { + return false, nil + } else if err != nil { + return false, fmt.Errorf("while creating %s: %w", Describe(obj), err) + } + return true, nil +} + // retryableApplyError reports whether a failed apply is worth re-sending. func retryableApplyError(err error) bool { // Apply is forced, so it never loses a field-ownership argument. A conflict @@ -271,6 +299,7 @@ func (c *Client) resourceFor(obj *unstructured.Unstructured) (dynamicResource, e // namespaced and cluster-scoped clients both satisfy it. type dynamicResource interface { Apply(ctx context.Context, name string, obj *unstructured.Unstructured, opts metav1.ApplyOptions, subresources ...string) (*unstructured.Unstructured, error) + Create(ctx context.Context, obj *unstructured.Unstructured, opts metav1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) Delete(ctx context.Context, name string, opts metav1.DeleteOptions, subresources ...string) error Get(ctx context.Context, name string, opts metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) } diff --git a/cmd/ate-setup/internal/kube/apply_test.go b/cmd/ate-setup/internal/kube/apply_test.go index ee73adf75f..58632de630 100644 --- a/cmd/ate-setup/internal/kube/apply_test.go +++ b/cmd/ate-setup/internal/kube/apply_test.go @@ -62,7 +62,8 @@ func TestDeletePathReportsUnparseableManifest(t *testing.T) { } } -// stubResource is a dynamicResource whose Apply returns queued results. +// stubResource is a dynamicResource whose Apply and Create return queued +// results. type stubResource struct { results []error attempts int @@ -76,6 +77,14 @@ func (s *stubResource) Apply(context.Context, string, *unstructured.Unstructured return nil, nil } +func (s *stubResource) Create(context.Context, *unstructured.Unstructured, metav1.CreateOptions, ...string) (*unstructured.Unstructured, error) { + s.attempts++ + if s.attempts <= len(s.results) { + return nil, s.results[s.attempts-1] + } + return nil, nil +} + func (s *stubResource) Delete(context.Context, string, metav1.DeleteOptions, ...string) error { return nil } @@ -158,3 +167,51 @@ func TestApplyWithRetryStopsOnCancellation(t *testing.T) { t.Errorf("apply attempted %d times, want 1", ri.attempts) } } + +func TestCreateWithRetry(t *testing.T) { + shortApplyBackoff(t) + alreadyExists := apierrors.NewAlreadyExists(schema.GroupResource{Resource: "configmaps"}, "ate-api-server-envvars") + unavailable := apierrors.NewServiceUnavailable("apiserver is starting") + + tests := []struct { + name string + results []error + cancelled bool + wantCreated bool + wantErr bool + wantAttempts int + }{ + {name: "created", wantCreated: true, wantAttempts: 1}, + {name: "exists", results: []error{alreadyExists}, wantAttempts: 1}, + {name: "created after a transient failure", results: []error{unavailable}, wantCreated: true, wantAttempts: 2}, + {name: "exists after a transient failure", results: []error{unavailable, alreadyExists}, wantAttempts: 2}, + { + name: "rejected", + results: []error{apierrors.NewInvalid(schema.GroupKind{Kind: "ConfigMap"}, "ate-api-server-envvars", nil)}, + wantErr: true, + wantAttempts: 1, + }, + // Retrying past a cancelled context turns ^C into an eight-second wait. + {name: "cancelled", results: []error{unavailable, unavailable}, cancelled: true, wantErr: true, wantAttempts: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if tt.cancelled { + cancel() + } + ri := &stubResource{results: tt.results} + created, err := createWithRetry(ctx, ri, testConfigMap()) + if (err != nil) != tt.wantErr { + t.Fatalf("createWithRetry() error = %v, wantErr %v", err, tt.wantErr) + } + if created != tt.wantCreated { + t.Errorf("createWithRetry() = %v, want %v", created, tt.wantCreated) + } + if ri.attempts != tt.wantAttempts { + t.Errorf("create attempted %d times, want %d", ri.attempts, tt.wantAttempts) + } + }) + } +} diff --git a/cmd/ate-setup/internal/steps/sandboxconfig.go b/cmd/ate-setup/internal/steps/sandboxconfig.go index 6f50e9973e..72c36d2e73 100644 --- a/cmd/ate-setup/internal/steps/sandboxconfig.go +++ b/cmd/ate-setup/internal/steps/sandboxconfig.go @@ -18,15 +18,14 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "slices" "strings" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/dynamic" @@ -52,7 +51,7 @@ func (e *Env) DeploySandboxConfig(ctx context.Context, createWithDefaultAnnotati // deploySandboxConfig is DeploySandboxConfig for the config that // checkSandboxConfig returned. -func (e *Env) deploySandboxConfig(ctx context.Context, sc sandboxConfig, createWithDefaultAnnotation bool) error { +func (e *Env) deploySandboxConfig(ctx context.Context, sc *unstructured.Unstructured, createWithDefaultAnnotation bool) error { log.Step("deploy_sandboxconfig") if err := e.EnsureCRDs(ctx); err != nil { @@ -65,27 +64,26 @@ func (e *Env) deploySandboxConfig(ctx context.Context, sc sandboxConfig, createW return err } - if sc.exists { - // An existing config may have been edited since, so it is left as it - // is. - log.Infof("SandboxConfig %s exists; leaving it and its annotations as they are", sc.name) - } else { - if err := e.Kube.ApplyBytes(ctx, sc.manifest); err != nil { - return err - } - log.Infof("created SandboxConfig %s", sc.name) - if createWithDefaultAnnotation { - patch, err := json.Marshal(map[string]any{ - "metadata": map[string]any{"annotations": map[string]any{atev1.SandboxConfigClassDefaultAnnotation: "true"}}, - }) - if err != nil { - return fmt.Errorf("while building the patch for SandboxConfig %s: %w", sc.name, err) - } - if _, err := e.sandboxConfigs().Patch(ctx, sc.name, types.MergePatchType, patch, metav1.PatchOptions{FieldManager: kube.FieldManager}); err != nil { - return fmt.Errorf("while annotating SandboxConfig %s: %w", sc.name, err) - } - log.Infof("annotated SandboxConfig %s %s=true", sc.name, atev1.SandboxConfigClassDefaultAnnotation) + if createWithDefaultAnnotation { + annotations := sc.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} } + annotations[atev1.SandboxConfigClassDefaultAnnotation] = "true" + sc.SetAnnotations(annotations) + } + // Create, not apply: an existing config may have been edited since, so it + // is left as it is. + created, err := e.Kube.CreateOne(ctx, sc) + if err != nil { + return err + } + if !created { + log.Infof("SandboxConfig %s exists; leaving it and its annotations as they are", sc.GetName()) + } else if createWithDefaultAnnotation { + log.Infof("created SandboxConfig %s with %s=true", sc.GetName(), atev1.SandboxConfigClassDefaultAnnotation) + } else { + log.Infof("created SandboxConfig %s", sc.GetName()) } // Checked after the create, against the config's real creation time and // annotations. @@ -93,55 +91,53 @@ func (e *Env) deploySandboxConfig(ctx context.Context, sc sandboxConfig, createW if err != nil { return err } - if warning := classDefaultWarning(configs, sc.name, atev1.SandboxClassGvisor); warning != "" { + if warning := classDefaultWarning(configs, sc.GetName(), atev1.SandboxClassGvisor); warning != "" { log.Warnf("%s", warning) } return nil } -// sandboxConfig is this build's gVisor SandboxConfig. -type sandboxConfig struct { - name string - // manifest is the config rendered with name. - manifest []byte - exists bool -} - // checkSandboxConfig returns this build's gVisor SandboxConfig, labeled with // the Substrate version that creates it. It fails when the version is not a -// valid label value, or when sandboxConfigExists fails. -func (e *Env) checkSandboxConfig(ctx context.Context) (sandboxConfig, error) { +// valid label value, or when requireSandboxConfigNotDeleting fails. +func (e *Env) checkSandboxConfig(ctx context.Context) (*unstructured.Unstructured, error) { version := e.SubstrateVersion() if errs := validation.IsValidLabelValue(version); len(errs) > 0 { - return sandboxConfig{}, fmt.Errorf("version %q is not a valid SandboxConfig label value: %s; set VERSION to one", version, strings.Join(errs, "; ")) + return nil, fmt.Errorf("version %q is not a valid SandboxConfig label value: %s; set VERSION to one", version, strings.Join(errs, "; ")) } tmpl := e.Cfg.Manifest("sandboxconfig-gvisor.yaml.tmpl") unnamed, err := render.Template(tmpl, nil, nil) if err != nil { - return sandboxConfig{}, err + return nil, err } name := sandboxConfigName(atev1.SandboxClassGvisor, unnamed) configs, err := e.listSandboxConfigs(ctx) if err != nil { - return sandboxConfig{}, err + return nil, err } - exists, err := sandboxConfigExists(configs, name) - if err != nil { - return sandboxConfig{}, err + if err := requireSandboxConfigNotDeleting(configs, name); err != nil { + return nil, err } manifest := render.Expand(string(unnamed), map[string]string{"SANDBOXCONFIG_NAME": name, "SUBSTRATE_VERSION": version}, nil) - return sandboxConfig{name: name, manifest: manifest, exists: exists}, nil + objs, err := kube.DecodeManifestBytes(manifest) + if err != nil { + return nil, err + } + if len(objs) != 1 { + return nil, fmt.Errorf("%s holds %d objects, want 1 SandboxConfig", tmpl, len(objs)) + } + return objs[0], nil } -// sandboxConfigExists reports whether configs hold the SandboxConfig named -// name. It fails when that config is being deleted, because applying it would -// only update the terminating object. -func sandboxConfigExists(configs []atev1.SandboxConfig, name string) (bool, error) { +// requireSandboxConfigNotDeleting fails when configs hold the SandboxConfig +// named name and it is being deleted, because the create would find the +// terminating object and leave it. +func requireSandboxConfigNotDeleting(configs []atev1.SandboxConfig, name string) error { i := slices.IndexFunc(configs, func(sc atev1.SandboxConfig) bool { return sc.Name == name }) if i >= 0 && !configs[i].DeletionTimestamp.IsZero() { - return false, fmt.Errorf("SandboxConfig %s is being deleted; wait until it is gone, then run this again", name) + return fmt.Errorf("SandboxConfig %s is being deleted; wait until it is gone, then run this again", name) } - return i >= 0, nil + return nil } // listSandboxConfigs returns the SandboxConfigs in the cluster. A cluster diff --git a/cmd/ate-setup/internal/steps/sandboxconfig_test.go b/cmd/ate-setup/internal/steps/sandboxconfig_test.go index a5fd34e4f7..e7d4b721f3 100644 --- a/cmd/ate-setup/internal/steps/sandboxconfig_test.go +++ b/cmd/ate-setup/internal/steps/sandboxconfig_test.go @@ -160,7 +160,7 @@ func TestClassDefaultWarning(t *testing.T) { } } -func TestSandboxConfigExists(t *testing.T) { +func TestRequireSandboxConfigNotDeleting(t *testing.T) { const name = "substrate-v2-gvisor" config := func(name string, deleting bool) atev1.SandboxConfig { sc := atev1.SandboxConfig{ObjectMeta: metav1.ObjectMeta{Name: name}} @@ -171,25 +171,21 @@ func TestSandboxConfigExists(t *testing.T) { } tests := []struct { - name string - configs []atev1.SandboxConfig - wantExists bool - wantErr bool + name string + configs []atev1.SandboxConfig + wantErr bool }{ {name: "fresh cluster"}, {name: "only other configs", configs: []atev1.SandboxConfig{config("substrate-v1-gvisor", false)}}, - {name: "exists", configs: []atev1.SandboxConfig{config(name, false)}, wantExists: true}, + {name: "exists", configs: []atev1.SandboxConfig{config(name, false)}}, {name: "being deleted", configs: []atev1.SandboxConfig{config(name, true)}, wantErr: true}, {name: "another config being deleted", configs: []atev1.SandboxConfig{config("substrate-v1-gvisor", true)}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - exists, err := sandboxConfigExists(tt.configs, name) + err := requireSandboxConfigNotDeleting(tt.configs, name) if (err != nil) != tt.wantErr { - t.Fatalf("sandboxConfigExists() error = %v, wantErr %v", err, tt.wantErr) - } - if exists != tt.wantExists { - t.Errorf("sandboxConfigExists() = %v, want %v", exists, tt.wantExists) + t.Fatalf("requireSandboxConfigNotDeleting() error = %v, wantErr %v", err, tt.wantErr) } }) } diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index ae193f25c5..995b0575b8 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -268,11 +268,14 @@ if [[ "${exists}" == "true" ]]; then log "SandboxConfig ${SANDBOXCONFIG_NAME} exists; leaving it and its annotations as they are." else log "Creating SandboxConfig ${SANDBOXCONFIG_NAME} from ${MANIFEST_TEMPLATE}..." - run_kubectl create -f - <<<"$(sed -e "s|\${SANDBOXCONFIG_NAME}|${SANDBOXCONFIG_NAME}|g" \ + rendered="$(sed -e "s|\${SANDBOXCONFIG_NAME}|${SANDBOXCONFIG_NAME}|g" \ -e "s|\${SUBSTRATE_VERSION}|${version}|g" <<<"${manifest}")" + # The annotation goes into the created object, so the config never exists + # without it. if [[ "${create_with_default_annotation}" == "true" ]]; then - run_kubectl annotate --overwrite sandboxconfig "${SANDBOXCONFIG_NAME}" "${DEFAULT_ANNOTATION}=true" + rendered="$(kubectl annotate --local -f - -o yaml "${DEFAULT_ANNOTATION}=true" <<<"${rendered}")" fi + run_kubectl create -f - <<<"${rendered}" fi # Checked again against the config's real creation time and annotations. microvm_configs="$(get_microvm_configs)"