From b1500c37f31001734d9ec8957b95c6ec07872141 Mon Sep 17 00:00:00 2001 From: jybao Date: Wed, 23 Sep 2026 22:21:55 +0000 Subject: [PATCH 01/10] ate-setup: add deploy/delete microvm-deps The micro-VM prerequisites -- the guest assets and the cluster-wide `microvm` SandboxConfig -- were reachable only as a side effect of `deploy benchmarks --sandbox-class=microvm`, which also brings up the whole locust stack. Everything else that needs them, the three micro-VM demos and any WorkerPool of your own, was told to run hack/install-microvm-deps.sh by hand. Make the cluster capable of running micro-VM workers a step of its own. The script is still invoked rather than reimplemented, for the reason recorded in differences.md: it drives asset assembly and object-store staging that are out of scope for the installer. What changes is that nothing outside the steps package has to name it. The benchmark paths now go through the same steps, so the two callers cannot drift apart. Fixes #1826 Signed-off-by: Noah West --- cmd/ate-setup/commands.md | 17 +- cmd/ate-setup/differences.md | 5 + cmd/ate-setup/internal/cmd/microvm.go | 60 +++++++ cmd/ate-setup/internal/cmd/microvm_test.go | 50 ++++++ .../demos/countermicrovm/countermicrovm.go | 8 +- .../demos/egressmicrovm/egressmicrovm.go | 6 +- cmd/ate-setup/internal/steps/benchmarks.go | 9 +- cmd/ate-setup/internal/steps/microvm.go | 55 ++++++ cmd/ate-setup/internal/steps/microvm_test.go | 161 ++++++++++++++++++ demos/counter/README.md | 5 +- 10 files changed, 359 insertions(+), 17 deletions(-) create mode 100644 cmd/ate-setup/internal/cmd/microvm.go create mode 100644 cmd/ate-setup/internal/cmd/microvm_test.go create mode 100644 cmd/ate-setup/internal/steps/microvm.go create mode 100644 cmd/ate-setup/internal/steps/microvm_test.go diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 267b790648..a42f686cde 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -116,12 +116,19 @@ that already names a manifest is used as written, and is not looked up. | `deploy podcertificate-controller` | (no shell equivalent) | | `deploy sandboxconfig` | (no shell equivalent) | | `deploy postgres` | (no shell equivalent) | +| `deploy microvm-deps` | `hack/install-microvm-deps.sh --install` | `deploy ate-system` is the whole control plane: CRDs, RBAC, the store, the apiserver, the controller, atenet, and atelet. It creates every `create` resource below on the way, so those subcommands are only needed to redo one on a running cluster. +`deploy microvm-deps` makes the cluster able to run micro-VM workers: it +assembles the guest assets, stages them in the object store the atelet pulls +from, and applies the cluster-wide `microvm` SandboxConfig. `deploy ate-system` +installs only `gvisor-default`, so this is needed before any micro-VM demo or +WorkerPool. `deploy benchmarks --sandbox-class=microvm` runs it for you. + ## Publish | `ate-setup` | `hack/install-ate.sh` | @@ -151,9 +158,15 @@ attaches the files to the draft release. | `delete ate-system` | `--delete-ate-system` | | `delete atenet` | `--delete-atenet` | | `delete all` | `--delete-all` | +| `delete microvm-deps` | `hack/install-microvm-deps.sh --delete` | `delete all` removes every registered demo and then the control plane. +`delete microvm-deps` removes only the cluster-wide `microvm` SandboxConfig; the +staged assets stay in the object store, which outlives any one install. It is +not part of `delete all`, since the SandboxConfig may belong to something other +than the install being removed. + ## Create Individual secrets and config that `deploy ate-system` creates automatically. @@ -210,9 +223,9 @@ See |---|---|---| | `deploy demo counter` | `--deploy-demo-counter` | A counter actor exercising snapshot, resume, and atenet ingress | | `deploy demo counter --with-external-volume [--storage-class NAME]` | `--deploy-demo-counter-with-external-volume` (`STORAGE_CLASS=NAME`) | The same, plus an external volume and a pre-seeded file to validate (requires `--preview=ExternalVolumes`). Run `setup csi` first and name the class it created, e.g. `csi-nfs-sc`; defaults to `standard` | -| `deploy demo counter-microvm` | `--deploy-demo-counter-microvm` | The counter demo on microVM workers. Run `hack/install-microvm-deps.sh --install` first | +| `deploy demo counter-microvm` | `--deploy-demo-counter-microvm` | The counter demo on microVM workers. Run `deploy microvm-deps` first | | `deploy demo egress` | `--deploy-demo-egress` | Egress policy enforcement through atenet; the actors trust the egress gateway's CA | -| `deploy demo egress-microvm` | `--deploy-demo-egress-microvm` | The same on microVM workers. Run `hack/install-microvm-deps.sh --install` first | +| `deploy demo egress-microvm` | `--deploy-demo-egress-microvm` | The same on microVM workers. Run `deploy microvm-deps` first | | `deploy demo jupyter` | `--deploy-demo-jupyter` | A Jupyter notebook server per actor, reached through atenet ingress | | `deploy demo sandbox` | `--deploy-demo-sandbox` | An on-demand sandbox actor driven by the sandbox client | | `deploy demo multi-template` | `--deploy-demo-multi-template` | Two ActorTemplates sharing one WorkerPool | diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 725594261f..e7a680d4ed 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -95,6 +95,11 @@ of scope for an installer: `benchmarking/deploy_locust.sh` and `hack/install-microvm-deps.sh`. They receive `Config.ScriptEnv()`, which reconstructs the environment the shell installer would have exported to them. +Both are fronted by commands, so nothing outside this package names a script: +`deploy`/`delete benchmarks` for the first, `deploy`/`delete microvm-deps` for +the second. That leaves the choice of whether to keep shelling out an internal +one. + `ko` is no longer asked to apply anything. The scripts ran `run_ko apply`, which made ko shell out to kubectl and forced the awkward `-- --context=` special case (only `apply`/`create`/`delete`/`run` accept args after `--`; `resolve` rejects diff --git a/cmd/ate-setup/internal/cmd/microvm.go b/cmd/ate-setup/internal/cmd/microvm.go new file mode 100644 index 0000000000..97a31140e3 --- /dev/null +++ b/cmd/ate-setup/internal/cmd/microvm.go @@ -0,0 +1,60 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/spf13/cobra" +) + +var deployMicroVMDepsCmd = &cobra.Command{ + Use: "microvm-deps", + Short: "Deploy the micro-VM prerequisites: guest assets and the microvm SandboxConfig", + Long: `Make the cluster able to run micro-VM workers. + +This assembles the micro-VM guest assets, stages them in the object store the +atelet pulls from, and applies the cluster-wide "microvm" SandboxConfig. + +"deploy ate-system" installs only the gvisor-default SandboxConfig, so run this +before anything that asks for micro-VM workers: "deploy demo counter-microvm", +"deploy demo egress-microvm", or a WorkerPool of your own. +"deploy benchmarks --sandbox-class=microvm" does it for you. + +Re-running is cheap: assets whose recorded versions still match are left alone.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return env.DeployMicroVMDeps(cmd.Context()) + }, +} + +var deleteMicroVMDepsCmd = &cobra.Command{ + Use: "microvm-deps", + Short: "Delete the cluster-wide microvm SandboxConfig", + Long: `Remove the cluster-wide "microvm" SandboxConfig. + +The staged assets are left in the object store: the bucket outlives any one +install, and staging them again is the slow part of "deploy microvm-deps". + +The SandboxConfig is cluster-wide, so check that nothing else is still asking +for micro-VM workers before removing it.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return env.DeleteMicroVMDeps(cmd.Context()) + }, +} + +func init() { + deployCmd.AddCommand(deployMicroVMDepsCmd) + deleteCmd.AddCommand(deleteMicroVMDepsCmd) +} diff --git a/cmd/ate-setup/internal/cmd/microvm_test.go b/cmd/ate-setup/internal/cmd/microvm_test.go new file mode 100644 index 0000000000..d675329834 --- /dev/null +++ b/cmd/ate-setup/internal/cmd/microvm_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "testing" + + "github.com/spf13/cobra" +) + +// The two commands share a name under different parents, which cobra allows +// and a mistake in init() would silently turn into one command registered +// twice under the same parent. +func TestMicroVMDepsIsRegisteredUnderBothVerbs(t *testing.T) { + for _, tc := range []struct { + parent *cobra.Command + want *cobra.Command + }{ + {deployCmd, deployMicroVMDepsCmd}, + {deleteCmd, deleteMicroVMDepsCmd}, + } { + t.Run(tc.parent.Name(), func(t *testing.T) { + found, _, err := tc.parent.Find([]string{"microvm-deps"}) + if err != nil { + t.Fatalf("%s microvm-deps: %v", tc.parent.Name(), err) + } + if found != tc.want { + t.Fatalf("%s microvm-deps resolved to %v, want the %s subcommand", + tc.parent.Name(), found.CommandPath(), tc.parent.Name()) + } + // Staging assets and deleting a cluster-wide SandboxConfig are + // both worth refusing on a typo rather than guessing at. + if err := found.Args(found, []string{"stray"}); err == nil { + t.Error("Args([stray]) = nil, want an error") + } + }) + } +} diff --git a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go index 52a5036e62..5b94a1b8c0 100644 --- a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go +++ b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go @@ -14,9 +14,9 @@ // Package countermicrovm installs the micro-VM variant of the counter demo. // -// It needs the cluster-wide `microvm` SandboxConfig that -// hack/install-microvm-deps.sh --install creates. It has no external-volume -// option: the CSI path is covered by demo-counter. +// It needs the cluster-wide `microvm` SandboxConfig that `deploy +// microvm-deps` creates. It has no external-volume option: the CSI path is +// covered by demo-counter. package countermicrovm import ( @@ -32,7 +32,7 @@ const namespace = "ate-demo-counter-microvm" func init() { demos.Register(&demos.Substrate{ DemoName: "demo-counter-microvm", - Short: "The counter demo on micro-VM workers (needs hack/install-microvm-deps.sh --install)", + Short: "The counter demo on micro-VM workers (needs `deploy microvm-deps`)", WorkerPoolManifest: "demos/counter/counter-microvm.yaml.tmpl", Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "counter-microvm"}}, Templates: []demos.SubstrateTemplate{{ diff --git a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go index 7c8146f494..613d3729d7 100644 --- a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go +++ b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go @@ -14,8 +14,8 @@ // Package egressmicrovm installs the micro-VM variant of the egress demo. // -// It needs the cluster-wide `microvm` SandboxConfig that -// hack/install-microvm-deps.sh --install creates. It is a demo of its own +// It needs the cluster-wide `microvm` SandboxConfig that `deploy +// microvm-deps` creates. It is a demo of its own // rather than a flag on demo-egress so that it appears in help and in the // `delete all` sweep, and so that both can be installed side by side: the // networking e2e suite runs against whichever the sandbox class under test @@ -35,7 +35,7 @@ const namespace = "ate-demo-egress-microvm" func init() { demos.Register(&demos.Substrate{ DemoName: "demo-egress-microvm", - Short: "Egress policy enforcement on micro-VM workers (needs hack/install-microvm-deps.sh --install)", + Short: "Egress policy enforcement on micro-VM workers (needs `deploy microvm-deps`)", WorkerPoolManifest: "demos/egress/egress-microvm.yaml.tmpl", Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "egress-microvm"}}, Templates: []demos.SubstrateTemplate{{ diff --git a/cmd/ate-setup/internal/steps/benchmarks.go b/cmd/ate-setup/internal/steps/benchmarks.go index 68c51d020c..5087d8a834 100644 --- a/cmd/ate-setup/internal/steps/benchmarks.go +++ b/cmd/ate-setup/internal/steps/benchmarks.go @@ -29,10 +29,7 @@ import ( // several other scripts (image builds, asset assembly, object-store staging) // that are out of scope for this command, so ate-setup shells out to them with // a translated argument list rather than reimplementing them. -const ( - deployLocustScript = "benchmarking/deploy_locust.sh" - installMicrovmDepScript = "hack/install-microvm-deps.sh" -) +const deployLocustScript = "benchmarking/deploy_locust.sh" // BenchmarkOptions shapes the benchmark WorkerPool. type BenchmarkOptions struct { @@ -74,7 +71,7 @@ func (e *Env) DeployBenchmarks(ctx context.Context, opts BenchmarkOptions) error // `deploy ate-system`, which only installs gvisor-default. The workloads // deploy references it by name and would fail if this were skipped. if opts.SandboxClass == config.SandboxClassMicrovm { - if err := e.runScript(ctx, installMicrovmDepScript, "--install"); err != nil { + if err := e.DeployMicroVMDeps(ctx); err != nil { return err } } @@ -118,7 +115,7 @@ func (e *Env) DeleteBenchmarks(ctx context.Context, opts BenchmarkOptions) error // gone. The script leaves it in place if another pool still uses it. // `delete all` tries again once every demo is gone. if opts.SandboxClass == config.SandboxClassMicrovm { - return e.runScript(ctx, installMicrovmDepScript, "--delete") + return e.DeleteMicroVMDeps(ctx) } return nil } diff --git a/cmd/ate-setup/internal/steps/microvm.go b/cmd/ate-setup/internal/steps/microvm.go new file mode 100644 index 0000000000..924c5fa17e --- /dev/null +++ b/cmd/ate-setup/internal/steps/microvm.go @@ -0,0 +1,55 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "context" + + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" +) + +// installMicrovmDepScript assembles the micro-VM guest assets, stages them in +// the object store the atelet pulls from, and applies the cluster-wide +// `microvm` SandboxConfig. It is invoked rather than reimplemented for the +// reason given above deployLocustScript: the asset assembly and object-store +// staging it drives are out of scope for this command. +const installMicrovmDepScript = "hack/install-microvm-deps.sh" + +// DeployMicroVMDeps installs the micro-VM prerequisites: the guest assets and +// the cluster-wide `microvm` SandboxConfig. +// +// `deploy ate-system` installs only the gvisor-default SandboxConfig, so +// anything that asks for micro-VM workers needs this first: the micro-VM +// demos, a micro-VM benchmark run, or a WorkerPool of your own. It is a step +// of its own rather than something each of those does for itself because +// making a cluster micro-VM capable is worth doing without also deploying a +// workload onto it. +// +// Re-running is cheap: the script leaves assets alone when their recorded +// versions still match, and re-applies the SandboxConfig either way. +func (e *Env) DeployMicroVMDeps(ctx context.Context) error { + log.Step("deploy_microvm_deps") + return e.runScript(ctx, installMicrovmDepScript, "--install") +} + +// DeleteMicroVMDeps removes the cluster-wide `microvm` SandboxConfig. +// +// The staged assets are deliberately left in the object store. The bucket +// outlives any one install, and re-staging is the slow half of +// DeployMicroVMDeps. +func (e *Env) DeleteMicroVMDeps(ctx context.Context) error { + log.Step("delete_microvm_deps") + return e.runScript(ctx, installMicrovmDepScript, "--delete") +} diff --git a/cmd/ate-setup/internal/steps/microvm_test.go b/cmd/ate-setup/internal/steps/microvm_test.go new file mode 100644 index 0000000000..df164b5227 --- /dev/null +++ b/cmd/ate-setup/internal/steps/microvm_test.go @@ -0,0 +1,161 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "context" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" +) + +// fakeMicrovmScript installs a stand-in for hack/install-microvm-deps.sh in a +// throwaway repository root and returns an Env pointing at it, plus a function +// reading back the lines it recorded. +// +// The script is the whole contract these steps have with the shell -- the +// exact flag and the directory it runs in -- so the test drives the real +// exec path rather than a seam around it. Only shell builtins are used: the +// script inherits Config.ScriptEnv(), which carries no PATH. +func fakeMicrovmScript(t *testing.T, exitCode string) (*Env, func() []string) { + t.Helper() + + root := t.TempDir() + record := filepath.Join(root, "record.txt") + script := filepath.Join(root, installMicrovmDepScript) + if err := os.MkdirAll(filepath.Dir(script), 0o755); err != nil { + t.Fatalf("creating the script directory: %v", err) + } + body := "#!/bin/sh\n{ pwd; printf '%s\\n' \"$@\"; } > " + record + "\nexit " + exitCode + "\n" + if err := os.WriteFile(script, []byte(body), 0o755); err != nil { + t.Fatalf("writing the fake script: %v", err) + } + + // The steps announce themselves on the installer's log; tests have no use + // for it. + log.SetOutput(io.Discard) + t.Cleanup(func() { log.SetOutput(os.Stdout) }) + + return &Env{Cfg: &config.Config{Root: root}}, func() []string { + t.Helper() + got, err := os.ReadFile(record) + if err != nil { + t.Fatalf("the script did not run: %v", err) + } + lines := strings.Split(strings.TrimSuffix(string(got), "\n"), "\n") + // macOS resolves TempDir through /private; compare what the shell saw + // against the same resolution. + if resolved, err := filepath.EvalSymlinks(root); err == nil { + lines[0] = strings.Replace(lines[0], resolved, root, 1) + } + return lines + } +} + +// Getting the flag wrong is the whole failure mode here: --install and +// --delete are opposites, and the script takes nothing else. +func TestMicroVMDepsStepsPassTheRightFlag(t *testing.T) { + for _, tc := range []struct { + name string + run func(*Env) error + want string + }{ + {"deploy", func(e *Env) error { return e.DeployMicroVMDeps(t.Context()) }, "--install"}, + {"delete", func(e *Env) error { return e.DeleteMicroVMDeps(t.Context()) }, "--delete"}, + } { + t.Run(tc.name, func(t *testing.T) { + env, recorded := fakeMicrovmScript(t, "0") + + if err := tc.run(env); err != nil { + t.Fatalf("running the step: %v", err) + } + + got := recorded() + // The script resolves manifests/ and hack/ relative to the + // directory it starts in, so running it from anywhere but the + // repository root would stage the wrong thing. + if got[0] != env.Cfg.Root { + t.Errorf("the script ran in %q, want the repository root %q", got[0], env.Cfg.Root) + } + if args := got[1:]; !slices.Equal(args, []string{tc.want}) { + t.Errorf("the script got %v, want [%s]", args, tc.want) + } + }) + } +} + +// A failed staging run has to stop the caller rather than leave it deploying +// workloads onto a cluster with no micro-VM assets. +func TestMicroVMDepsStepsReportAFailedScript(t *testing.T) { + env, _ := fakeMicrovmScript(t, "1") + + err := env.DeployMicroVMDeps(t.Context()) + if err == nil { + t.Fatal("DeployMicroVMDeps() = nil, want the script's failure") + } + if !strings.Contains(err.Error(), installMicrovmDepScript) { + t.Errorf("error = %v, want it to name %s", err, installMicrovmDepScript) + } +} + +// Deploying benchmarks onto micro-VM has to bring the SandboxConfig with it: +// the workloads reference it by name. Only the micro-VM half is exercised here +// -- deploy_locust.sh is not stubbed, so the step fails right after. +func TestDeployBenchmarksInstallsMicroVMDeps(t *testing.T) { + env, recorded := fakeMicrovmScript(t, "0") + + // The error is deploy_locust.sh missing, which is the next thing to run. + _ = env.DeployBenchmarks(t.Context(), BenchmarkOptions{ + WorkerCount: 1, + SandboxClass: config.SandboxClassMicrovm, + }) + + if args := recorded()[1:]; !slices.Equal(args, []string{"--install"}) { + t.Errorf("the script got %v, want [--install]", args) + } +} + +// Confirm the opposite: a gvisor benchmark run must not touch the cluster-wide +// micro-VM SandboxConfig. +func TestDeleteBenchmarksLeavesMicroVMDepsAloneForGvisor(t *testing.T) { + env, _ := fakeMicrovmScript(t, "0") + + _ = env.DeleteBenchmarks(t.Context(), BenchmarkOptions{ + WorkerCount: 1, + SandboxClass: config.SandboxClassGvisor, + }) + + if _, err := os.Stat(filepath.Join(env.Cfg.Root, "record.txt")); !os.IsNotExist(err) { + t.Error("the micro-VM script ran for a gvisor teardown") + } +} + +func TestMicroVMDepsStepsAreSeparableFromContext(t *testing.T) { + // A cancelled context must stop the script rather than run it and discard + // the result; the caller uses cancellation to abort a long staging run. + env, _ := fakeMicrovmScript(t, "0") + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + if err := env.DeployMicroVMDeps(ctx); err == nil { + t.Error("DeployMicroVMDeps() with a cancelled context = nil, want an error") + } +} diff --git a/demos/counter/README.md b/demos/counter/README.md index 98bb4b5329..40da79a08a 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -48,10 +48,11 @@ Run it and follow the printed next steps: KIND_CLUSTER_NAME= ./hack/run-microvm-demo-kind.sh ``` -On a cluster that already has the microVM deps (`hack/install-microvm-deps.sh`), -deploy directly instead: +To deploy onto an existing cluster instead, install the microVM deps and then +the demo: ```bash +go run ./cmd/ate-setup deploy microvm-deps go run ./cmd/ate-setup deploy demo counter-microvm ``` From 4a6c9acad2682e980eda9b55a1519146dcc476a9 Mon Sep 17 00:00:00 2001 From: Noah West Date: Wed, 30 Sep 2026 15:38:50 -0700 Subject: [PATCH 02/10] ate-setup: port microvm-deps installation from shell to Go Replace the shell implementation of hack/install-microvm-deps.sh and the helper scripts under hack/microvm-assets/ (assemble.sh, stage-to-gcs.sh, stage-to-rustfs.sh) with native Go in cmd/ate-setup/internal/steps/microvm.go. DeployMicroVMDeps now resolves the target architecture and asset version stamp in Go, streams and extracts the kata-static .tar.zst archive and Cloud Hypervisor binary, verifies the bundled virtiofsd version when runnable on the host, stages the assets to the Kind rustfs bucket or GKE GCS bucket, and renders and server-side-applies the cluster-wide microvm SandboxConfig. DeleteMicroVMDeps removes the SandboxConfig by name while leaving bucket assets intact. Keep hack/install-microvm-deps.sh as a flag-translation shim calling ate-setup deploy/delete microvm-deps and remove the obsolete scripts under hack/microvm-assets/. Signed-off-by: Noah West --- .github/workflows/pr-workflow.yaml | 9 +- benchmarking/automation/README.md | 2 +- benchmarking/automation/orchestrator.py | 4 +- cmd/ate-setup/differences.md | 22 +- cmd/ate-setup/internal/steps/microvm.go | 594 +++++++++++++++++- cmd/ate-setup/internal/steps/microvm_test.go | 545 +++++++++++++--- .../internal/third_party/kata/PROVENANCE.md | 2 +- docs/dev/microvm-local.md | 22 +- hack/install-microvm-deps.sh | 191 +----- hack/microvm-assets/README.md | 35 +- hack/microvm-assets/assemble.sh | 144 ----- hack/microvm-assets/stage-to-gcs.sh | 46 -- hack/microvm-assets/stage-to-rustfs.sh | 102 --- hack/run-microvm-demo.sh | 10 +- .../microvm/sandboxconfig-microvm.yaml.tmpl | 19 +- 15 files changed, 1108 insertions(+), 639 deletions(-) delete mode 100755 hack/microvm-assets/assemble.sh delete mode 100755 hack/microvm-assets/stage-to-gcs.sh delete mode 100755 hack/microvm-assets/stage-to-rustfs.sh diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 4584f1057b..61d6115a81 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -259,12 +259,13 @@ jobs: uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: # Assembling the assets (download kata-static + cloud-hypervisor + the - # prebuilt virtiofsd — all downloads on amd64) is fully pinned by assemble.sh, - # so key the cache on its hash. The push-to-main run populates the - # default-branch cache PRs inherit; the weekly schedule refreshes it. + # prebuilt virtiofsd — all downloads on amd64) is fully pinned by + # cmd/ate-setup/internal/steps/microvm.go, so key the cache on its hash. + # The push-to-main run populates the default-branch cache PRs inherit; + # the weekly schedule refreshes it. # run-microvm-demo-kind.sh skips assembling when these are present. path: bin/microvm-assets/amd64 - key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }} + key: microvm-assets-amd64-${{ hashFiles('cmd/ate-setup/internal/steps/microvm.go') }} - name: Enable KVM # Grant the runner access to /dev/kvm so create-kind-cluster.sh mounts it into # the node and labels it for the micro-VM sandbox class. diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 7f36ddf5be..de34c358b5 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -28,7 +28,7 @@ the router capacity benchmark — see runner's calls pass [authorization](../../docs/authorization.md), and sets `ATE_INGRESS_ALLOWED_SPIFFE_IDS` to `benchmarking/benchmark-runner` for the Envoy ingress router. For a `microvm` test the orchestrator also - runs `hack/install-microvm-deps.sh --install` between the two, which + runs `go run ./cmd/ate-setup deploy microvm-deps` between the two, which stages kata + cloud-hypervisor + virtiofsd assets to the cluster's object store bucket and applies the cluster-wide `microvm` SandboxConfig. For a `nighthawk-ingress` test the orchestrator additionally patches the diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index f60e90e455..fd72c59ebe 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -424,14 +424,14 @@ def install_microvm_deps() -> None: """Stage kata/cloud-hypervisor assets and apply the cluster-wide microvm SandboxConfig. Required before a microvm WorkerPool can run actors; must run after deploy_substrate() (which installs the CRDs).""" - run(["hack/install-microvm-deps.sh", "--install"]) + run(["go", "run", "./cmd/ate-setup", "deploy", "microvm-deps"]) def teardown_microvm_deps() -> None: """Remove the microvm SandboxConfig. Must run before teardown_substrate(), which deletes the SandboxConfig CRD (and would prevent this from succeeding via kubectl).""" - run_no_check(["hack/install-microvm-deps.sh", "--delete"]) + run_no_check(["go", "run", "./cmd/ate-setup", "delete", "microvm-deps"]) def deploy_workloads( diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index e7a680d4ed..12b0af9dfb 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -85,20 +85,20 @@ Still required, and why: | `ko` | building and publishing images (`ko resolve` only); not used with `--image-repo` | | `go` | locating the pinned `ko` tool, exactly as `hack/run-tool.sh` does; not used with `--image-repo` | | `git` | the `git describe` version stamp passed to ko; not used with `--image-repo` | -| `docker` | the kind CSI setup (`docker exec` into the node) and the claude-code-multiplex workload build | -| `gcloud` | GKE `get-credentials`, only when `PROJECT_ID` is set and no context was given | +| `docker` | the kind CSI setup (`docker exec` into the node), staging micro-VM assets into Kind's `rustfs`, and the claude-code-multiplex workload build | +| `gcloud` | GKE `get-credentials` (when `PROJECT_ID` is set and no context was given) and `gcloud storage` when staging micro-VM assets on GKE | | `bash` | sourcing `.ate-dev-env.sh` | -Two shell scripts are still invoked rather than reimplemented, because they -orchestrate image builds, asset assembly, and object-store staging that are out -of scope for an installer: `benchmarking/deploy_locust.sh` and -`hack/install-microvm-deps.sh`. They receive `Config.ScriptEnv()`, which -reconstructs the environment the shell installer would have exported to them. +One shell script is still invoked rather than reimplemented, because it +orchestrates benchmark image builds and Locust stack deployment: +`benchmarking/deploy_locust.sh`. It receives `Config.ScriptEnv()`, which +reconstructs the environment the shell installer would have exported to it, and +is fronted by `deploy`/`delete benchmarks`. -Both are fronted by commands, so nothing outside this package names a script: -`deploy`/`delete benchmarks` for the first, `deploy`/`delete microvm-deps` for -the second. That leaves the choice of whether to keep shelling out an internal -one. +The micro-VM asset assembly (`kata-static` `.tar.zst` extraction and +`cloud-hypervisor` download), object-store staging (`rustfs` on Kind, GCS on +GKE), and `microvm` `SandboxConfig` lifecycle live in `deploy`/`delete +microvm-deps`, with `hack/install-microvm-deps.sh` kept as a translation shim. `ko` is no longer asked to apply anything. The scripts ran `run_ko apply`, which made ko shell out to kubectl and forced the awkward `-- --context=` special case diff --git a/cmd/ate-setup/internal/steps/microvm.go b/cmd/ate-setup/internal/steps/microvm.go index 924c5fa17e..147f29d15f 100644 --- a/cmd/ate-setup/internal/steps/microvm.go +++ b/cmd/ate-setup/internal/steps/microvm.go @@ -15,41 +15,589 @@ package steps import ( + "archive/tar" + "bufio" + "cmp" "context" + "crypto/sha256" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" + "time" + "github.com/klauspost/compress/zstd" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/render" +) + +const ( + // MicroVMStageTimeout bounds the wait for the in-cluster rustfs Deployment + // and bucket-init Job on Kind. + MicroVMStageTimeout = 300 * time.Second + + defaultKataVersion = "4.1.0" + defaultCloudHypervisorVersion = "v53.0" + // virtiofsdVersion is the version bundled inside kata-static for + // defaultKataVersion (not env-overridable; verified before stamping). + virtiofsdVersion = "1.14.0" + + assetStampFile = ".asset-versions" + defaultKindMicroVMBucket = "ate-snapshots" + microvmSandboxConfigTemplate = "manifests/microvm/sandboxconfig-microvm.yaml.tmpl" + microvmSandboxConfigName = "microvm" + + // Keep in sync with manifests/ate-install/kind/rustfs.yaml. + rustfsAWSCLIImage = "amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73" + + kataSharePrefix = "opt/kata/share/kata-containers/" + kataKernelSymlink = "opt/kata/share/kata-containers/vmlinux.container" + kataRootfsSymlink = "opt/kata/share/kata-containers/kata-containers.img" + kataLibexecVirtiofsd = "opt/kata/libexec/virtiofsd" + + maxMicroVMAssetBytes int64 = 8 << 30 ) -// installMicrovmDepScript assembles the micro-VM guest assets, stages them in -// the object store the atelet pulls from, and applies the cluster-wide -// `microvm` SandboxConfig. It is invoked rather than reimplemented for the -// reason given above deployLocustScript: the asset assembly and object-store -// staging it drives are out of scope for this command. -const installMicrovmDepScript = "hack/install-microvm-deps.sh" +var microvmAssets = []string{"cloud-hypervisor", "virtiofsd", "vmlinux", "rootfs.img"} + +var sandboxConfigGVK = schema.GroupVersionKind{ + Group: "ate.dev", + Version: "v1alpha1", + Kind: "SandboxConfig", +} + +// microvmReleaseBaseURL is overridden in unit tests to point at an httptest.Server. +var microvmReleaseBaseURL = "https://github.com" // DeployMicroVMDeps installs the micro-VM prerequisites: the guest assets and // the cluster-wide `microvm` SandboxConfig. -// -// `deploy ate-system` installs only the gvisor-default SandboxConfig, so -// anything that asks for micro-VM workers needs this first: the micro-VM -// demos, a micro-VM benchmark run, or a WorkerPool of your own. It is a step -// of its own rather than something each of those does for itself because -// making a cluster micro-VM capable is worth doing without also deploying a -// workload onto it. -// -// Re-running is cheap: the script leaves assets alone when their recorded -// versions still match, and re-applies the SandboxConfig either way. func (e *Env) DeployMicroVMDeps(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } log.Step("deploy_microvm_deps") - return e.runScript(ctx, installMicrovmDepScript, "--install") + + arch := e.microvmArch() + if _, err := cloudHypervisorAssetName(arch); err != nil { + return err + } + bucket, err := e.microVMBucket() + if err != nil { + return err + } + outDir := cmp.Or(e.getenv("OUT"), e.Cfg.Path("bin", "microvm-assets", arch)) + kataVer := cmp.Or(e.getenv("KATA_VER"), defaultKataVersion) + chVer := cmp.Or(e.getenv("CH_VER"), defaultCloudHypervisorVersion) + + wantStamp := microvmAssetStamp(arch, kataVer, chVer) + needAssemble, stale := microvmAssetsNeedAssemble(outDir, wantStamp) + if stale { + log.Infof("Asset set in %s is stale (version stamp mismatch); re-assembling.", outDir) + } + if needAssemble { + log.Infof("Assembling micro-VM assets into %s (ARCH=%s)...", outDir, arch) + if err := assembleMicroVMAssets(ctx, outDir, arch, kataVer, chVer); err != nil { + return err + } + } else { + log.Infof("Assets already present in %s; skipping assemble.", outDir) + } + + if e.Cfg.Kind { + log.Infof("Staging assets to in-cluster rustfs bucket %s (kata-assets/)...", bucket) + if err := e.stageMicroVMAssetsToRustfs(ctx, outDir, bucket); err != nil { + return err + } + } else { + log.Infof("Uploading assets to gs://%s/kata-assets/ ...", bucket) + if err := e.stageMicroVMAssetsToGCS(ctx, outDir, bucket); err != nil { + return err + } + } + + log.Infof("Applying microvm SandboxConfig from %s...", microvmSandboxConfigTemplate) + rendered, err := render.Template( + e.Cfg.Path("manifests", "microvm", "sandboxconfig-microvm.yaml.tmpl"), + map[string]string{ + "BUCKET_SCHEME": e.microVMBucketScheme(), + "ASSET_BUCKET_NAME": bucket, + }, + nil, + ) + if err != nil { + return err + } + if err := e.Kube.ApplyBytes(ctx, rendered); err != nil { + return err + } + log.Infof("Done. ActorTemplates must reference this SandboxConfig by name (sandboxConfig.configName: microvm).") + return nil } -// DeleteMicroVMDeps removes the cluster-wide `microvm` SandboxConfig. -// -// The staged assets are deliberately left in the object store. The bucket -// outlives any one install, and re-staging is the slow half of -// DeployMicroVMDeps. +// DeleteMicroVMDeps removes the cluster-wide `microvm` SandboxConfig, leaving +// staged bucket assets in place. func (e *Env) DeleteMicroVMDeps(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } log.Step("delete_microvm_deps") - return e.runScript(ctx, installMicrovmDepScript, "--delete") + log.Infof("Deleting microvm SandboxConfig...") + + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(sandboxConfigGVK) + obj.SetName(microvmSandboxConfigName) + if err := e.Kube.DeleteOne(ctx, obj); err != nil { + return err + } + if bucket, err := e.microVMBucket(); err == nil { + log.Infof("Done. (Bucket assets at %s://%s/kata-assets/ left in place.)", e.microVMBucketScheme(), bucket) + } else { + log.Infof("Done.") + } + return nil +} + +// microVMBucket returns the bucket where micro-VM guest assets are staged: the +// shared rustfs bucket on Kind, and -assets on GKE (where atelet +// has read access only to the asset bucket). +func (e *Env) microVMBucket() (string, error) { + if e.Cfg.Kind { + if e.Cfg.BucketName != "" { + return e.Cfg.BucketName, nil + } + return defaultKindMicroVMBucket, nil + } + if e.Cfg.BucketName == "" { + return "", fmt.Errorf("BUCKET_NAME (--storage-bucket-name or config.storage.bucketName) must be set to stage micro-VM assets on non-Kind clusters, or pass --skip-microvm-deps") + } + return e.Cfg.BucketName + "-assets", nil +} + +// microVMBucketScheme returns the URL scheme for the node's object-store +// plugin: s3 on Kind (rustfs) and gs on GKE. +func (e *Env) microVMBucketScheme() string { + if e.Cfg.Kind { + return "s3" + } + return "gs" +} + +func (e *Env) microvmArch() string { + if arch := e.getenv("ARCH"); arch != "" { + return arch + } + if e.Cfg != nil && e.Cfg.KODefaultPlatforms != "" { + if _, arch, ok := strings.Cut(e.Cfg.KODefaultPlatforms, "/"); ok { + return arch + } + return e.Cfg.KODefaultPlatforms + } + return runtime.GOARCH +} + +func (e *Env) getenv(key string) string { + if v := os.Getenv(key); v != "" { + return v + } + if e.Cfg == nil { + return "" + } + prefix := key + "=" + for _, kv := range e.Cfg.ScriptEnv() { + if val, ok := strings.CutPrefix(kv, prefix); ok { + return val + } + } + return "" +} + +func cloudHypervisorAssetName(arch string) (string, error) { + switch arch { + case "arm64": + return "cloud-hypervisor-static-aarch64", nil + case "amd64": + return "cloud-hypervisor-static", nil + default: + return "", fmt.Errorf("unsupported ARCH=%s", arch) + } +} + +func microvmAssetStamp(arch, kataVer, chVer string) string { + return fmt.Sprintf("arch=%s\nkata=%s\ncloud-hypervisor=%s\nvirtiofsd=%s\n", + arch, kataVer, chVer, virtiofsdVersion) +} + +func checkMicroVMAssets(outDir string) error { + for _, f := range microvmAssets { + if fi, err := os.Stat(filepath.Join(outDir, f)); err != nil || !fi.Mode().IsRegular() { + return fmt.Errorf("missing asset %s/%s", outDir, f) + } + } + return nil +} + +func microvmAssetsNeedAssemble(outDir, wantStamp string) (needAssemble, stale bool) { + if err := checkMicroVMAssets(outDir); err != nil { + return true, false + } + got, err := os.ReadFile(filepath.Join(outDir, assetStampFile)) + if err != nil || string(got) != wantStamp { + return true, true + } + return false, false +} + +// assembleMicroVMAssets downloads the Kata static tarball and Cloud Hypervisor +// binary for arch into outDir, verifies the bundled virtiofsd version when +// runnable on the host, and writes the .asset-versions stamp last. +func assembleMicroVMAssets(ctx context.Context, outDir, arch, kataVer, chVer string) error { + chAsset, err := cloudHypervisorAssetName(arch) + if err != nil { + return err + } + if err := os.MkdirAll(outDir, 0o755); err != nil { + return fmt.Errorf("while creating %s: %w", outDir, err) + } + + // Remove any existing stamp before mutating files in outDir so a failed + // run leaves an unstamped directory that the next run will re-assemble. + stampPath := filepath.Join(outDir, assetStampFile) + _ = os.Remove(stampPath) + + workDir, err := os.MkdirTemp(outDir, ".assemble-*") + if err != nil { + return fmt.Errorf("while creating temporary directory in %s: %w", outDir, err) + } + defer func() { _ = os.RemoveAll(workDir) }() + + baseURL := strings.TrimSuffix(microvmReleaseBaseURL, "/") + kataURL := fmt.Sprintf("%s/kata-containers/kata-containers/releases/download/%s/kata-static-%s-%s.tar.zst", + baseURL, kataVer, kataVer, arch) + log.Infof(">> Downloading kata-static %s (%s)...", kataVer, arch) + if err := downloadAndExtractKataAssets(ctx, kataURL, workDir, outDir); err != nil { + return err + } + + chURL := fmt.Sprintf("%s/cloud-hypervisor/cloud-hypervisor/releases/download/%s/%s", baseURL, chVer, chAsset) + log.Infof(">> Downloading cloud-hypervisor %s (%s)...", chVer, chAsset) + if err := downloadToFile(ctx, chURL, filepath.Join(outDir, "cloud-hypervisor"), 0o755); err != nil { + return err + } + + log.Infof(">> Assets assembled in %s:", outDir) + if err := checkMicroVMAssets(outDir); err != nil { + return err + } + if err := verifyVirtiofsdVersion(ctx, filepath.Join(outDir, "virtiofsd"), kataVer, virtiofsdVersion); err != nil { + return err + } + if err := os.WriteFile(stampPath, []byte(microvmAssetStamp(arch, kataVer, chVer)), 0o644); err != nil { + return fmt.Errorf("while writing %s: %w", stampPath, err) + } + + log.Infof(">> sha256 (paste all four into the per-arch block in %s):", microvmSandboxConfigTemplate) + for _, f := range microvmAssets { + file, err := os.Open(filepath.Join(outDir, f)) + if err != nil { + return err + } + h := sha256.New() + _, copyErr := io.Copy(h, file) + _ = file.Close() + if copyErr != nil { + return copyErr + } + log.Infof("%x %s", h.Sum(nil), f) + } + return nil +} + +func downloadAndExtractKataAssets(ctx context.Context, kataURL, workDir, outDir string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, kataURL, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("while downloading %s: %w", kataURL, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("while downloading %s: unexpected HTTP status %s", kataURL, resp.Status) + } + + links, err := extractKataTarZst(ctx, resp.Body, workDir) + if err != nil { + return fmt.Errorf("while extracting %s: %w", kataURL, err) + } + + for _, item := range []struct { + archivePath string + outName string + mode fs.FileMode + }{ + {archivePath: kataKernelSymlink, outName: "vmlinux", mode: 0o644}, + {archivePath: kataRootfsSymlink, outName: "rootfs.img", mode: 0o644}, + {archivePath: kataLibexecVirtiofsd, outName: "virtiofsd", mode: 0o755}, + } { + resolved := item.archivePath + for range 8 { + target, ok := links[resolved] + if !ok { + break + } + resolved = target + } + src := filepath.Join(workDir, filepath.FromSlash(resolved)) + if fi, err := os.Stat(src); err != nil || !fi.Mode().IsRegular() { + return fmt.Errorf("while resolving %s in kata archive: missing regular file %s", item.archivePath, resolved) + } + if err := os.Chmod(src, item.mode); err != nil { + return err + } + if err := os.Rename(src, filepath.Join(outDir, item.outName)); err != nil { + return err + } + } + return nil +} + +// extractKataTarZst decompresses a zstd-compressed tar stream, writes regular +// files under opt/kata/share/kata-containers/ and opt/kata/libexec/virtiofsd +// into workDir, and returns a map of symlink paths to their cleaned archive +// targets. +func extractKataTarZst(ctx context.Context, r io.Reader, workDir string) (map[string]string, error) { + zr, err := zstd.NewReader(bufio.NewReader(r)) + if err != nil { + return nil, fmt.Errorf("creating zstd reader: %w", err) + } + defer zr.Close() + + links := map[string]string{} + tr := tar.NewReader(zr) + var totalBytes int64 + for { + if err := ctx.Err(); err != nil { + return nil, err + } + hdr, err := tr.Next() + if err == io.EOF { + return links, nil + } + if err != nil { + return nil, err + } + + name := strings.TrimPrefix(path.Clean(hdr.Name), "/") + if name == "." || name == "" { + continue + } + if !filepath.IsLocal(filepath.FromSlash(name)) { + return nil, fmt.Errorf("tar entry %q escapes the extraction directory", hdr.Name) + } + if !strings.HasPrefix(name, kataSharePrefix) && name != kataLibexecVirtiofsd { + continue + } + + switch hdr.Typeflag { + case tar.TypeReg: + totalBytes += hdr.Size + if totalBytes > maxMicroVMAssetBytes { + return nil, fmt.Errorf("kata archive exceeds %d-byte limit", maxMicroVMAssetBytes) + } + dst := filepath.Join(workDir, filepath.FromSlash(name)) + if err := writeStreamToFile(dst, 0o600, tr); err != nil { + return nil, err + } + case tar.TypeSymlink: + if path.IsAbs(hdr.Linkname) { + return nil, fmt.Errorf("tar symlink %q has absolute target %q", hdr.Name, hdr.Linkname) + } + target := path.Clean(path.Join(path.Dir(name), hdr.Linkname)) + if !filepath.IsLocal(filepath.FromSlash(target)) { + return nil, fmt.Errorf("tar symlink %q target %q escapes the extraction directory", hdr.Name, hdr.Linkname) + } + links[name] = target + } + } +} + +func writeStreamToFile(dstPath string, mode fs.FileMode, r io.Reader) error { + if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { + return err + } + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return err + } + n, err := io.Copy(f, io.LimitReader(r, maxMicroVMAssetBytes+1)) + closeErr := f.Close() + if err != nil { + return err + } + if n > maxMicroVMAssetBytes { + return fmt.Errorf("extracted file exceeds %d-byte limit", maxMicroVMAssetBytes) + } + return closeErr +} + +func downloadToFile(ctx context.Context, url, dstPath string, mode fs.FileMode) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("while downloading %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("while downloading %s: unexpected HTTP status %s", url, resp.Status) + } + return writeStreamToFile(dstPath, mode, resp.Body) +} + +func verifyVirtiofsdVersion(ctx context.Context, virtiofsdPath, kataVer, wantVersion string) error { + out, err := exec.CommandContext(ctx, virtiofsdPath, "--version").Output() + if err != nil { + return ctx.Err() + } + firstLine, _, _ := strings.Cut(string(out), "\n") + fields := strings.Fields(firstLine) + if len(fields) < 2 { + return nil + } + if got := fields[1]; got != wantVersion { + return fmt.Errorf("kata %s bundles virtiofsd %s, not %s: update virtiofsdVersion", kataVer, got, wantVersion) + } + log.Infof("virtiofsd %s", fields[1]) + return nil +} + +func (e *Env) stageMicroVMAssetsToRustfs(ctx context.Context, outDir, bucket string) error { + if _, err := exec.LookPath("docker"); err != nil { + return fmt.Errorf("'docker' is required to run the S3 client but was not found in PATH: %w", err) + } + + ns := e.Namespace() + timeout := e.Cfg.WaitTimeout(MicroVMStageTimeout) + log.Infof(">> Waiting for rustfs in namespace %s...", ns) + if err := e.Kube.RolloutStatus(ctx, kube.KindDeployment, ns, "rustfs", timeout); err != nil { + return err + } + if err := e.waitJobComplete(ctx, ns, "rustfs-bucket-init", timeout); err != nil { + return err + } + + node, err := e.kindNodeName(ctx) + if err != nil { + return err + } + svc, err := e.Kube.Typed.CoreV1().Services(ns).Get(ctx, "rustfs", metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("while getting service %s/rustfs: %w", ns, err) + } + if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == corev1.ClusterIPNone { + return fmt.Errorf("service %s/rustfs has no ClusterIP", ns) + } + endpoint := fmt.Sprintf("http://%s:9000", svc.Spec.ClusterIP) + + awsCLI := func(stdin io.Reader, args ...string) error { + dockerArgs := append([]string{ + "run", "--rm", "-i", + "--network", "container:" + node, + "-e", "AWS_ACCESS_KEY_ID=" + cmp.Or(e.getenv("AWS_ACCESS_KEY_ID"), "rustfsadmin"), + "-e", "AWS_SECRET_ACCESS_KEY=" + cmp.Or(e.getenv("AWS_SECRET_ACCESS_KEY"), "rustfsadmin"), + "-e", "AWS_REGION=" + cmp.Or(e.getenv("AWS_REGION"), "us-east-1"), + "-e", "AWS_ENDPOINT_URL=" + endpoint, + rustfsAWSCLIImage, + }, args...) + return e.runCmd(ctx, stdin, "docker", dockerArgs...) + } + + log.Infof(">> Uploading assets to s3://%s/kata-assets/ via %s (netns of %s)...", bucket, endpoint, node) + for _, f := range microvmAssets { + log.Infof(" %s", f) + file, err := os.Open(filepath.Join(outDir, f)) + if err != nil { + return err + } + err = awsCLI(file, "s3", "cp", "-", fmt.Sprintf("s3://%s/kata-assets/%s", bucket, f)) + _ = file.Close() + if err != nil { + return fmt.Errorf("while uploading %s to rustfs: %w", f, err) + } + } + + log.Infof(">> Done. Verify:") + return awsCLI(nil, "s3", "ls", fmt.Sprintf("s3://%s/kata-assets/", bucket)) +} + +func (e *Env) waitJobComplete(ctx context.Context, namespace, name string, timeout time.Duration) error { + defer log.Elapsed(time.Now(), fmt.Sprintf("wait for job/%s", name)) + return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { + job, err := e.Kube.Typed.BatchV1().Jobs(namespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + for _, cond := range job.Status.Conditions { + if cond.Type == batchv1.JobComplete && cond.Status == corev1.ConditionTrue { + return true, nil + } + if cond.Type == batchv1.JobFailed && cond.Status == corev1.ConditionTrue { + return false, fmt.Errorf("job %s/%s failed: %s", namespace, name, cond.Message) + } + } + return false, nil + }) +} + +func (e *Env) stageMicroVMAssetsToGCS(ctx context.Context, outDir, bucket string) error { + var projectFlag []string + if e.Cfg.ProjectID != "" { + projectFlag = []string{"--project=" + e.Cfg.ProjectID} + } + + for _, f := range microvmAssets { + log.Infof(" %s", f) + args := append(append([]string{"storage", "cp"}, projectFlag...), + filepath.Join(outDir, f), fmt.Sprintf("gs://%s/kata-assets/%s", bucket, f)) + if err := e.runCmd(ctx, nil, "gcloud", args...); err != nil { + return fmt.Errorf("while uploading %s to GCS: %w", f, err) + } + } + + log.Infof(">> Done. Verify:") + lsArgs := append(append([]string{"storage", "ls"}, projectFlag...), fmt.Sprintf("gs://%s/kata-assets/", bucket)) + return e.runCmd(ctx, nil, "gcloud", lsArgs...) +} + +func (e *Env) runCmd(ctx context.Context, stdin io.Reader, name string, args ...string) error { + cmd := exec.CommandContext(ctx, name, args...) + if e.Cfg != nil { + cmd.Dir = e.Cfg.Root + cmd.Env = e.Cfg.ScriptEnv() + } + cmd.Stdin = stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() } diff --git a/cmd/ate-setup/internal/steps/microvm_test.go b/cmd/ate-setup/internal/steps/microvm_test.go index df164b5227..11195c1d3e 100644 --- a/cmd/ate-setup/internal/steps/microvm_test.go +++ b/cmd/ate-setup/internal/steps/microvm_test.go @@ -15,147 +15,538 @@ package steps import ( + "archive/tar" + "bytes" "context" + "errors" "io" + "net/http" + "net/http/httptest" "os" "path/filepath" - "slices" + "runtime" "strings" "testing" + "time" + + "github.com/klauspost/compress/zstd" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/render" ) -// fakeMicrovmScript installs a stand-in for hack/install-microvm-deps.sh in a -// throwaway repository root and returns an Env pointing at it, plus a function -// reading back the lines it recorded. -// -// The script is the whole contract these steps have with the shell -- the -// exact flag and the directory it runs in -- so the test drives the real -// exec path rather than a seam around it. Only shell builtins are used: the -// script inherits Config.ScriptEnv(), which carries no PATH. -func fakeMicrovmScript(t *testing.T, exitCode string) (*Env, func() []string) { +func silenceStepLog(t *testing.T) { t.Helper() + log.SetOutput(io.Discard) + t.Cleanup(func() { log.SetOutput(os.Stdout) }) +} - root := t.TempDir() - record := filepath.Join(root, "record.txt") - script := filepath.Join(root, installMicrovmDepScript) - if err := os.MkdirAll(filepath.Dir(script), 0o755); err != nil { - t.Fatalf("creating the script directory: %v", err) +func TestMicroVMConfigResolution(t *testing.T) { + t.Run("ARCH env beats KO_DEFAULTPLATFORMS and GOARCH", func(t *testing.T) { + t.Setenv("ARCH", "arm64") + e := &Env{Cfg: &config.Config{KODefaultPlatforms: "linux/amd64"}} + if got := e.microvmArch(); got != "arm64" { + t.Errorf("microvmArch() = %q, want arm64", got) + } + }) + + t.Run("KO_DEFAULTPLATFORMS suffix beats GOARCH", func(t *testing.T) { + t.Setenv("ARCH", "") + e := &Env{Cfg: &config.Config{KODefaultPlatforms: "linux/arm64"}} + if got := e.microvmArch(); got != "arm64" { + t.Errorf("microvmArch() = %q, want arm64", got) + } + }) + + t.Run("falls back to runtime.GOARCH", func(t *testing.T) { + t.Setenv("ARCH", "") + e := &Env{Cfg: &config.Config{}} + if got := e.microvmArch(); got != runtime.GOARCH { + t.Errorf("microvmArch() = %q, want %q", got, runtime.GOARCH) + } + }) +} + +func TestMicroVMAssetsNeedAssemble(t *testing.T) { + stamp := microvmAssetStamp("amd64", defaultKataVersion, defaultCloudHypervisorVersion) + const wantStamp = "arch=amd64\nkata=4.1.0\ncloud-hypervisor=v53.0\nvirtiofsd=1.14.0\n" + if stamp != wantStamp { + t.Fatalf("microvmAssetStamp() =\n%q\nwant\n%q", stamp, wantStamp) } - body := "#!/bin/sh\n{ pwd; printf '%s\\n' \"$@\"; } > " + record + "\nexit " + exitCode + "\n" - if err := os.WriteFile(script, []byte(body), 0o755); err != nil { - t.Fatalf("writing the fake script: %v", err) + + writeAllAssets := func(t *testing.T, dir string) { + t.Helper() + for _, f := range microvmAssets { + if err := os.WriteFile(filepath.Join(dir, f), []byte(f), 0o644); err != nil { + t.Fatal(err) + } + } } - // The steps announce themselves on the installer's log; tests have no use - // for it. - log.SetOutput(io.Discard) - t.Cleanup(func() { log.SetOutput(os.Stdout) }) + t.Run("missing file triggers assemble without stale warning", func(t *testing.T) { + dir := t.TempDir() + for _, f := range microvmAssets[:3] { + if err := os.WriteFile(filepath.Join(dir, f), []byte(f), 0o644); err != nil { + t.Fatal(err) + } + } + need, stale := microvmAssetsNeedAssemble(dir, stamp) + if !need || stale { + t.Errorf("microvmAssetsNeedAssemble() = (%v, %v), want (true, false)", need, stale) + } + }) - return &Env{Cfg: &config.Config{Root: root}}, func() []string { - t.Helper() - got, err := os.ReadFile(record) + t.Run("missing stamp with all files present reports stale", func(t *testing.T) { + dir := t.TempDir() + writeAllAssets(t, dir) + need, stale := microvmAssetsNeedAssemble(dir, stamp) + if !need || !stale { + t.Errorf("microvmAssetsNeedAssemble() = (%v, %v), want (true, true)", need, stale) + } + }) + + t.Run("mismatched stamp reports stale", func(t *testing.T) { + dir := t.TempDir() + writeAllAssets(t, dir) + oldStamp := microvmAssetStamp("amd64", "4.0.0", defaultCloudHypervisorVersion) + if err := os.WriteFile(filepath.Join(dir, assetStampFile), []byte(oldStamp), 0o644); err != nil { + t.Fatal(err) + } + need, stale := microvmAssetsNeedAssemble(dir, stamp) + if !need || !stale { + t.Errorf("microvmAssetsNeedAssemble() = (%v, %v), want (true, true)", need, stale) + } + }) + + t.Run("matching stamp skips assemble", func(t *testing.T) { + dir := t.TempDir() + writeAllAssets(t, dir) + if err := os.WriteFile(filepath.Join(dir, assetStampFile), []byte(stamp), 0o644); err != nil { + t.Fatal(err) + } + need, stale := microvmAssetsNeedAssemble(dir, stamp) + if need || stale { + t.Errorf("microvmAssetsNeedAssemble() = (%v, %v), want (false, false)", need, stale) + } + }) +} + +type tarTestEntry struct { + name string + typeflag byte + linkname string + body string + mode int64 +} + +func buildKataTarZst(t *testing.T, entries []tarTestEntry) []byte { + t.Helper() + var buf bytes.Buffer + zw, err := zstd.NewWriter(&buf) + if err != nil { + t.Fatal(err) + } + tw := tar.NewWriter(zw) + for _, e := range entries { + mode := e.mode + if mode == 0 { + mode = 0o644 + } + hdr := &tar.Header{ + Name: e.name, + Typeflag: e.typeflag, + Linkname: e.linkname, + Size: int64(len(e.body)), + Mode: mode, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if e.typeflag == tar.TypeReg { + if _, err := tw.Write([]byte(e.body)); err != nil { + t.Fatal(err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func validKataArchive(t *testing.T, virtiofsdScript string) []byte { + t.Helper() + // Place the vmlinux.container symlink BEFORE its target file to verify + // extraction does not depend on entry order in the tar stream. + return buildKataTarZst(t, []tarTestEntry{ + { + name: "opt/kata/share/kata-containers/vmlinux.container", + typeflag: tar.TypeSymlink, + linkname: "vmlinux-6.12.47-173", + }, + { + name: "opt/kata/share/kata-containers/vmlinux-6.12.47-173", + typeflag: tar.TypeReg, + body: "fake-kernel-bytes", + }, + { + name: "opt/kata/share/kata-containers/kata-containers-ubuntu.img", + typeflag: tar.TypeReg, + body: "fake-rootfs-bytes", + }, + { + name: "opt/kata/share/kata-containers/kata-containers.img", + typeflag: tar.TypeSymlink, + linkname: "kata-containers-ubuntu.img", + }, + { + name: "opt/kata/libexec/virtiofsd", + typeflag: tar.TypeReg, + body: virtiofsdScript, + mode: 0o755, + }, + { + name: "opt/kata/bin/qemu-system-x86_64", + typeflag: tar.TypeReg, + body: "ignored-qemu-binary", + }, + }) +} + +func serveFakeReleases(t *testing.T, archive []byte, chBody string, failCH bool) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/kata-containers/kata-containers/releases/download/4.1.0/kata-static-4.1.0-amd64.tar.zst": + _, _ = w.Write(archive) + case "/cloud-hypervisor/cloud-hypervisor/releases/download/v53.0/cloud-hypervisor-static": + if failCH { + http.Error(w, "boom", http.StatusInternalServerError) + return + } + _, _ = w.Write([]byte(chBody)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + origURL := microvmReleaseBaseURL + microvmReleaseBaseURL = srv.URL + t.Cleanup(func() { microvmReleaseBaseURL = origURL }) +} + +func TestAssembleMicroVMAssets(t *testing.T) { + silenceStepLog(t) + + virtiofsdBody := "#!/bin/sh\necho 'virtiofsd " + virtiofsdVersion + "'\n" + const chBody = "fake-cloud-hypervisor-binary" + serveFakeReleases(t, validKataArchive(t, virtiofsdBody), chBody, false) + + outDir := t.TempDir() + if err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.1.0", "v53.0"); err != nil { + t.Fatalf("assembleMicroVMAssets() = %v", err) + } + + for name, wantBody := range map[string]string{ + "vmlinux": "fake-kernel-bytes", + "rootfs.img": "fake-rootfs-bytes", + "virtiofsd": virtiofsdBody, + "cloud-hypervisor": chBody, + } { + got, err := os.ReadFile(filepath.Join(outDir, name)) + if err != nil { + t.Fatalf("reading %s: %v", name, err) + } + if string(got) != wantBody { + t.Errorf("%s content = %q, want %q", name, got, wantBody) + } + } + + for _, execName := range []string{"virtiofsd", "cloud-hypervisor"} { + fi, err := os.Stat(filepath.Join(outDir, execName)) if err != nil { - t.Fatalf("the script did not run: %v", err) + t.Fatal(err) } - lines := strings.Split(strings.TrimSuffix(string(got), "\n"), "\n") - // macOS resolves TempDir through /private; compare what the shell saw - // against the same resolution. - if resolved, err := filepath.EvalSymlinks(root); err == nil { - lines[0] = strings.Replace(lines[0], resolved, root, 1) + if fi.Mode().Perm()&0o111 == 0 { + t.Errorf("%s mode = %v, want executable bit set", execName, fi.Mode()) } - return lines + } + + stamp, err := os.ReadFile(filepath.Join(outDir, assetStampFile)) + if err != nil { + t.Fatalf("reading %s: %v", assetStampFile, err) + } + if want := microvmAssetStamp("amd64", "4.1.0", "v53.0"); string(stamp) != want { + t.Errorf("stamp = %q, want %q", stamp, want) + } +} + +func TestAssembleMicroVMAssetsClearsStampOnFailure(t *testing.T) { + silenceStepLog(t) + + serveFakeReleases(t, validKataArchive(t, "#!/bin/sh\necho 'virtiofsd "+virtiofsdVersion+"'\n"), "", true) + + outDir := t.TempDir() + stampPath := filepath.Join(outDir, assetStampFile) + if err := os.WriteFile(stampPath, []byte("pre-existing-stamp\n"), 0o644); err != nil { + t.Fatal(err) + } + + if err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.1.0", "v53.0"); err == nil { + t.Fatal("assembleMicroVMAssets() succeeded when cloud-hypervisor download returned 500") + } + if _, err := os.Stat(stampPath); !errors.Is(err, os.ErrNotExist) { + t.Errorf("stamp file after failed assemble: err = %v, want ErrNotExist", err) + } +} + +func TestAssembleMicroVMAssetsRejectsVirtiofsdVersionMismatch(t *testing.T) { + silenceStepLog(t) + + serveFakeReleases(t, validKataArchive(t, "#!/bin/sh\necho 'virtiofsd 1.13.3'\n"), "ch", false) + + outDir := t.TempDir() + err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.1.0", "v53.0") + if err == nil || !strings.Contains(err.Error(), "1.13.3") { + t.Fatalf("assembleMicroVMAssets() = %v, want virtiofsd 1.13.3 mismatch error", err) + } + if _, statErr := os.Stat(filepath.Join(outDir, assetStampFile)); !errors.Is(statErr, os.ErrNotExist) { + t.Errorf("stamp written despite virtiofsd mismatch: %v", statErr) } } -// Getting the flag wrong is the whole failure mode here: --install and -// --delete are opposites, and the script takes nothing else. -func TestMicroVMDepsStepsPassTheRightFlag(t *testing.T) { +func TestExtractKataTarZstRejectsTraversal(t *testing.T) { for _, tc := range []struct { - name string - run func(*Env) error - want string + name string + entry tarTestEntry }{ - {"deploy", func(e *Env) error { return e.DeployMicroVMDeps(t.Context()) }, "--install"}, - {"delete", func(e *Env) error { return e.DeleteMicroVMDeps(t.Context()) }, "--delete"}, + { + name: "dotdot path", + entry: tarTestEntry{ + name: "opt/kata/share/kata-containers/../../../../../etc/passwd", + typeflag: tar.TypeReg, + body: "bad", + }, + }, + { + name: "symlink escaping workDir", + entry: tarTestEntry{ + name: "opt/kata/share/kata-containers/vmlinux.container", + typeflag: tar.TypeSymlink, + linkname: "../../../../../../etc/passwd", + }, + }, + { + name: "absolute symlink target", + entry: tarTestEntry{ + name: "opt/kata/share/kata-containers/vmlinux.container", + typeflag: tar.TypeSymlink, + linkname: "/etc/passwd", + }, + }, } { t.Run(tc.name, func(t *testing.T) { - env, recorded := fakeMicrovmScript(t, "0") - - if err := tc.run(env); err != nil { - t.Fatalf("running the step: %v", err) + data := buildKataTarZst(t, []tarTestEntry{tc.entry}) + _, err := extractKataTarZst(t.Context(), bytes.NewReader(data), t.TempDir()) + if err == nil { + t.Fatal("extractKataTarZst() succeeded on escaping entry, want error") } + }) + } +} - got := recorded() - // The script resolves manifests/ and hack/ relative to the - // directory it starts in, so running it from anywhere but the - // repository root would stage the wrong thing. - if got[0] != env.Cfg.Root { - t.Errorf("the script ran in %q, want the repository root %q", got[0], env.Cfg.Root) +func TestMicroVMSandboxConfigTemplate(t *testing.T) { + tmplPath := filepath.Join(repoRoot(t), microvmSandboxConfigTemplate) + for _, tc := range []struct { + scheme string + bucket string + }{ + {scheme: "gs", bucket: "my-test-bucket"}, + {scheme: "s3", bucket: defaultKindMicroVMBucket}, + } { + t.Run(tc.scheme, func(t *testing.T) { + rendered, err := render.Template(tmplPath, map[string]string{ + "BUCKET_SCHEME": tc.scheme, + "ASSET_BUCKET_NAME": tc.bucket, + }, nil) + if err != nil { + t.Fatalf("render.Template(%s) = %v", tmplPath, err) + } + if strings.Contains(string(rendered), "${") { + t.Errorf("rendered manifest still contains unexpanded placeholder:\n%s", rendered) } - if args := got[1:]; !slices.Equal(args, []string{tc.want}) { - t.Errorf("the script got %v, want [%s]", args, tc.want) + wantURL := tc.scheme + "://" + tc.bucket + "/kata-assets/cloud-hypervisor" + if !strings.Contains(string(rendered), wantURL) { + t.Errorf("rendered manifest missing expected bucket URL %q:\n%s", wantURL, rendered) + } + objs, err := kube.DecodeManifestBytes(rendered) + if err != nil { + t.Fatalf("DecodeManifestBytes() = %v", err) + } + if len(objs) != 1 || objs[0].GetKind() != "SandboxConfig" || objs[0].GetName() != microvmSandboxConfigName { + t.Errorf("decoded objects = %v, want single SandboxConfig/%s", objs, microvmSandboxConfigName) } }) } } -// A failed staging run has to stop the caller rather than leave it deploying -// workloads onto a cluster with no micro-VM assets. -func TestMicroVMDepsStepsReportAFailedScript(t *testing.T) { - env, _ := fakeMicrovmScript(t, "1") +func TestDeployMicroVMDepsRequiresBucketOnNonKind(t *testing.T) { + silenceStepLog(t) + env := &Env{Cfg: &config.Config{Root: t.TempDir(), Kind: false}} err := env.DeployMicroVMDeps(t.Context()) - if err == nil { - t.Fatal("DeployMicroVMDeps() = nil, want the script's failure") + if err == nil || !strings.Contains(err.Error(), "BUCKET_NAME") { + t.Fatalf("DeployMicroVMDeps() on non-Kind without BucketName = %v, want BUCKET_NAME error", err) } - if !strings.Contains(err.Error(), installMicrovmDepScript) { - t.Errorf("error = %v, want it to name %s", err, installMicrovmDepScript) +} + +func TestMicroVMBucket(t *testing.T) { + for _, tc := range []struct { + name string + cfg config.Config + wantBucket string + wantScheme string + wantErr bool + }{ + { + name: "Kind defaults to ate-snapshots over s3", + cfg: config.Config{Kind: true}, + wantBucket: defaultKindMicroVMBucket, + wantScheme: "s3", + }, + { + name: "Kind honors explicit BucketName", + cfg: config.Config{Kind: true, BucketName: "custom-kind-bucket"}, + wantBucket: "custom-kind-bucket", + wantScheme: "s3", + }, + { + name: "Non-Kind requires explicit BucketName", + cfg: config.Config{Kind: false}, + wantScheme: "gs", + wantErr: true, + }, + { + name: "Non-Kind uses -assets over gs", + cfg: config.Config{Kind: false, BucketName: "my-gke-bucket"}, + wantBucket: "my-gke-bucket-assets", + wantScheme: "gs", + }, + } { + t.Run(tc.name, func(t *testing.T) { + env := &Env{Cfg: &tc.cfg} + gotBucket, err := env.microVMBucket() + if (err != nil) != tc.wantErr { + t.Fatalf("microVMBucket() error = %v, wantErr %v", err, tc.wantErr) + } + if gotBucket != tc.wantBucket { + t.Errorf("microVMBucket() = %q, want %q", gotBucket, tc.wantBucket) + } + if gotScheme := env.microVMBucketScheme(); gotScheme != tc.wantScheme { + t.Errorf("microVMBucketScheme() = %q, want %q", gotScheme, tc.wantScheme) + } + }) } } +func TestWaitJobComplete(t *testing.T) { + silenceStepLog(t) + + t.Run("returns nil when JobComplete is True", func(t *testing.T) { + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: "rustfs-bucket-init"}, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + {Type: batchv1.JobComplete, Status: corev1.ConditionTrue}, + }, + }, + } + e := &Env{Kube: fakeKube(t, job)} + if err := e.waitJobComplete(t.Context(), NamespaceAteSystem, "rustfs-bucket-init", time.Second); err != nil { + t.Errorf("waitJobComplete() = %v, want nil", err) + } + }) + + t.Run("returns error immediately when JobFailed is True", func(t *testing.T) { + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: "rustfs-bucket-init"}, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + {Type: batchv1.JobFailed, Status: corev1.ConditionTrue, Message: "BackoffLimitExceeded"}, + }, + }, + } + e := &Env{Kube: fakeKube(t, job)} + err := e.waitJobComplete(t.Context(), NamespaceAteSystem, "rustfs-bucket-init", time.Second) + if err == nil || !strings.Contains(err.Error(), "BackoffLimitExceeded") { + t.Errorf("waitJobComplete() = %v, want BackoffLimitExceeded error", err) + } + }) +} + // Deploying benchmarks onto micro-VM has to bring the SandboxConfig with it: -// the workloads reference it by name. Only the micro-VM half is exercised here -// -- deploy_locust.sh is not stubbed, so the step fails right after. +// the workloads reference it by name. func TestDeployBenchmarksInstallsMicroVMDeps(t *testing.T) { - env, recorded := fakeMicrovmScript(t, "0") + silenceStepLog(t) - // The error is deploy_locust.sh missing, which is the next thing to run. - _ = env.DeployBenchmarks(t.Context(), BenchmarkOptions{ + // Passing an unsupported ARCH makes DeployMicroVMDeps return an error + // immediately, proving DeployBenchmarks invoked it before deploy_locust.sh. + t.Setenv("ARCH", "unsupported-arch") + env := &Env{Cfg: &config.Config{Root: t.TempDir(), Kind: true}} + + err := env.DeployBenchmarks(t.Context(), BenchmarkOptions{ WorkerCount: 1, SandboxClass: config.SandboxClassMicrovm, }) - - if args := recorded()[1:]; !slices.Equal(args, []string{"--install"}) { - t.Errorf("the script got %v, want [--install]", args) + if err == nil || !strings.Contains(err.Error(), "unsupported ARCH=unsupported-arch") { + t.Fatalf("DeployBenchmarks() = %v, want error from DeployMicroVMDeps", err) } } // Confirm the opposite: a gvisor benchmark run must not touch the cluster-wide // micro-VM SandboxConfig. func TestDeleteBenchmarksLeavesMicroVMDepsAloneForGvisor(t *testing.T) { - env, _ := fakeMicrovmScript(t, "0") + silenceStepLog(t) - _ = env.DeleteBenchmarks(t.Context(), BenchmarkOptions{ + root := t.TempDir() + locust := filepath.Join(root, deployLocustScript) + if err := os.MkdirAll(filepath.Dir(locust), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(locust, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + + // Kube is nil: if DeleteBenchmarks called DeleteMicroVMDeps for gvisor, it + // would dereference e.Kube and panic. + env := &Env{Cfg: &config.Config{Root: root}} + if err := env.DeleteBenchmarks(t.Context(), BenchmarkOptions{ WorkerCount: 1, SandboxClass: config.SandboxClassGvisor, - }) - - if _, err := os.Stat(filepath.Join(env.Cfg.Root, "record.txt")); !os.IsNotExist(err) { - t.Error("the micro-VM script ran for a gvisor teardown") + }); err != nil { + t.Fatalf("DeleteBenchmarks(gvisor) = %v, want nil", err) } } func TestMicroVMDepsStepsAreSeparableFromContext(t *testing.T) { - // A cancelled context must stop the script rather than run it and discard - // the result; the caller uses cancellation to abort a long staging run. - env, _ := fakeMicrovmScript(t, "0") + silenceStepLog(t) + + env := &Env{Cfg: &config.Config{Root: t.TempDir(), Kind: true}} ctx, cancel := context.WithCancel(t.Context()) cancel() if err := env.DeployMicroVMDeps(ctx); err == nil { t.Error("DeployMicroVMDeps() with a cancelled context = nil, want an error") } + if err := env.DeleteMicroVMDeps(ctx); err == nil { + t.Error("DeleteMicroVMDeps() with a cancelled context = nil, want an error") + } } diff --git a/cmd/ateworker-microvm/internal/third_party/kata/PROVENANCE.md b/cmd/ateworker-microvm/internal/third_party/kata/PROVENANCE.md index d75a1d5d1e..23a3cc902f 100644 --- a/cmd/ateworker-microvm/internal/third_party/kata/PROVENANCE.md +++ b/cmd/ateworker-microvm/internal/third_party/kata/PROVENANCE.md @@ -6,7 +6,7 @@ everything under it. - **Upstream:** github.com/kata-containers/kata-containers - **Version:** tag `3.31.0`. The microVM demo's runtime assets track a newer kata - (see `hack/microvm-assets/assemble.sh` `KATA_VER`, currently `4.3.1`); the four + (see `cmd/ate-setup/internal/steps/microvm.go` `defaultKataVersion`, currently `4.3.1`); the four vendored `.proto` files are byte-identical between `3.31.0` and `4.3.1` (verified 2026-10-23), so the generated Go needs no re-vendor. - **License:** Apache-2.0 — `./LICENSE` is the upstream license verbatim (mirrored to diff --git a/docs/dev/microvm-local.md b/docs/dev/microvm-local.md index 66aba29851..329bf8b650 100644 --- a/docs/dev/microvm-local.md +++ b/docs/dev/microvm-local.md @@ -147,23 +147,7 @@ limactl stop docker-nested limactl start docker-nested ``` -### 3. Assemble the arm64 assets inside the Lima VM - -Assemble the arm64 assets in the guest because macOS does not ship `zstd` by default: - -```sh -limactl shell docker-nested - -cd # visible via the writable home mount -./hack/microvm-assets/assemble.sh -exit -``` - -The script downloads about 590 MB and takes a minute or two, leaving ~285 MB in -`bin/microvm-assets/arm64/`. That directory is shared with the host through the -home mount — the demo script will find the assets there and skip re-assembling. - -### 4. Point the Docker CLI at Lima and bring everything up (on macOS) +### 3. Point the Docker CLI at Lima and bring everything up (on macOS) ```sh export DOCKER_HOST="unix://${HOME}/.lima/docker-nested/sock/docker.sock" @@ -175,6 +159,10 @@ cd ./hack/run-microvm-demo-kind.sh ``` +The first run of `run-microvm-demo-kind.sh` downloads about 590 MB of guest +runtime assets and extracts ~285 MB into `bin/microvm-assets/arm64/`; later runs +reuse that directory when the version stamp matches. + Verify as in Option A, step 4. ## Trying it out diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index acfe705d23..2de37b27d4 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -13,58 +13,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -# Install (or delete) the cluster-wide micro-VM (kata + cloud-hypervisor) -# dependencies. This is opt-in on top of hack/install-ate.sh --deploy-ate-system, -# which only installs the default gVisor SandboxConfig. Used by: -# * hack/run-microvm-demo.sh (before applying the counter-microvm demo) -# * benchmarking/automation/orchestrator.py (before deploying a microvm -# benchmark workload) -# -# On --install: assembles the asset set (assemble.sh; skipped if OUT already -# has them), stages the assets under kata-assets/ to the cluster's object -# store bucket (rustfs on kind, GCS on GKE), and applies the cluster-wide -# `microvm` SandboxConfig referencing those assets. Every asset sha256 is -# pinned in the manifest, so the apply only substitutes the bucket name. # -# On --delete: removes the SandboxConfig from the cluster. The delete is best -# effort. A SandboxConfig that a WorkerPool uses carries a protection -# finalizer, which atecontroller releases only once no pool's status names -# the config. So the script leaves the config in place if any WorkerPool -# still uses it. Otherwise it waits a bounded time for the finalizer to go. -# Bucket contents are left alone (they're inert until a SandboxConfig points -# at them, and re-staging is cheap on next install). +# Installs or deletes the cluster-wide micro-VM (kata + cloud-hypervisor) +# dependencies. # -# Like the other hack scripts, this sources .ate-dev-env.sh for the cluster / -# registry / bucket settings unless NO_DEV_ENV is set. +# This is a translation shim: cmd/ate-setup carries out the action (`deploy +# microvm-deps` / `delete microvm-deps`), and this script only maps the +# historical `--install` and `--delete` flags onto those commands so existing +# scripts and CI jobs keep working unchanged. # -# Env (most come from .ate-dev-env.sh): -# BUCKET_NAME object store bucket for snapshots (default: ate-snapshots). On -# kind the assets share it. On GKE they go to BUCKET_NAME-assets, -# which tools/setup-gcp creates, so nodes can read assets -# without being able to read snapshots. -# KUBECTL_CONTEXT (optional) kube context; threaded into kubectl. -# PROJECT_ID (optional) GCP project for the GCS asset upload (GKE path). -# ARCH target arch (default: from KO_DEFAULTPLATFORMS, else host arch). -# OUT asset dir (default: $PWD/bin/microvm-assets/$ARCH, gitignored). -# ATE_INSTALL_KIND "true" for the kind path (stage assets to rustfs); default -# false uploads assets to GCS. +# The environment variables the installer reads -- BUCKET_NAME, KUBECTL_CONTEXT, +# PROJECT_ID, KO_DEFAULTPLATFORMS, ARCH, OUT, ATE_INSTALL_KIND, NO_DEV_ENV -- +# and .ate-dev-env.sh are read by ate-setup directly. set -o errexit -o nounset -o pipefail ROOT="$(git rev-parse --show-toplevel)" cd "${ROOT}" -# Source the environment (cluster, registry, bucket) like the other hack scripts; -# callers set NO_DEV_ENV to skip this and use kind defaults. -if [[ -r .ate-dev-env.sh ]] && [[ -z "${NO_DEV_ENV:-}" ]]; then - source .ate-dev-env.sh -fi - -BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" -KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" -ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" - usage() { cat <&2; usage; exit 1 ;; esac shift @@ -94,135 +60,4 @@ if [[ -z "${action}" ]]; then exit 1 fi -# kubectl falls back to localhost:8080 when neither --context nor a kubeconfig -# current-context is set, which surfaces mid-install as a confusing "connection -# refused" from the apply -- after the assets have already been assembled and -# staged. Resolve the target cluster up front instead. -if [[ -z "${KUBECTL_CONTEXT}" ]] && ! kubectl config current-context >/dev/null 2>&1; then - echo "Error: no kube context to target: KUBECTL_CONTEXT is empty and the" >&2 - echo " kubeconfig has no current-context." >&2 - echo " Set KUBECTL_CONTEXT (e.g. in .ate-dev-env.sh) or run:" >&2 - echo " kubectl config use-context " >&2 - exit 1 -fi - -# ANSI color codes for prettier output (mirrors hack/install-ate.sh). -COLOR_CYAN='\033[1;36m' -COLOR_RESET='\033[0m' -log() { - echo -e "${COLOR_CYAN}[install-microvm-deps]: $*${COLOR_RESET}" -} - -run_kubectl() { - kubectl ${KUBECTL_CONTEXT:+--context="${KUBECTL_CONTEXT}"} "$@" -} - -MANIFEST_TEMPLATE="manifests/microvm/sandboxconfig-microvm.yaml.tmpl" - -# The asset URLs' scheme: the node's object-store plugin serves only its own -# backend's, which is S3 (rustfs) on kind and GCS on GKE. -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - BUCKET_SCHEME="s3" - ASSET_BUCKET_NAME="${BUCKET_NAME}" -else - BUCKET_SCHEME="gs" - ASSET_BUCKET_NAME="${BUCKET_NAME}-assets" -fi - -if [[ "${action}" == "delete" ]]; then - # A WorkerPool that uses the config holds it. The delete would only mark - # the config Terminating, and `kubectl delete` would wait for it without - # end. A Terminating config also cannot be recreated under its name. So no - # delete is issued at all. The config is left in place and the pools are - # named instead. A cluster without the WorkerPool CRD has no users. - users="$(run_kubectl get workerpools --all-namespaces \ - -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name} {.status.sandboxClasses[*].configRef.name}{"\n"}{end}' 2>/dev/null \ - | awk '{ for (i = 2; i <= NF; i++) if ($i == "microvm") { print $1; break } }' || true)" - if [[ -n "${users}" ]]; then - echo "Warning: the microvm SandboxConfig is left in place; these WorkerPools" >&2 - echo " still use it:" >&2 - while IFS= read -r pool; do echo " ${pool}" >&2; done <<<"${users}" - echo " Delete them, or point their spec.sandboxClasses[].configRef at" >&2 - echo " another microvm SandboxConfig, and run this again." >&2 - exit 0 - fi - log "Deleting microvm SandboxConfig..." - # Delete by name rather than by manifest so a missing/edited template file - # doesn't block cleanup on an older cluster. atecontroller releases the - # protection finalizer within a reconcile. The timeout turns a missing - # atecontroller into an error instead of a hang. - if ! run_kubectl delete --ignore-not-found --timeout=60s sandboxconfig microvm; then - echo "Error: the microvm SandboxConfig did not finish deleting. It stays" >&2 - echo " Terminating until atecontroller releases its protection" >&2 - echo " finalizer; check that atecontroller is running, then run this" >&2 - echo " again." >&2 - exit 1 - fi - log "Done. (Bucket assets at ${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/ left in place.)" - exit 0 -fi - -# --- install ---------------------------------------------------------------- - -# Target arch: match the images' platform (KO_DEFAULTPLATFORMS is set by -# .ate-dev-env.sh on GKE and by the kind wrapper); fall back to the host arch. -if [[ -z "${ARCH:-}" ]]; then - if [[ -n "${KO_DEFAULTPLATFORMS:-}" ]]; then - ARCH="${KO_DEFAULTPLATFORMS##*/}" - else - ARCH="$(go env GOARCH)" - fi -fi -OUT="${OUT:-${ROOT}/bin/microvm-assets/$ARCH}" - -# --- 1. assets: assemble (if missing or stale) ----------------------------- -need_assemble=false -for f in cloud-hypervisor virtiofsd vmlinux rootfs.img; do - if [[ ! -f "${OUT}/${f}" ]]; then - need_assemble=true - break - fi -done -# Presence alone is not enough. The filenames don't change when a version pin -# moves, so an asset dir assembled before a bump looks complete while holding the old -# bytes — we'd then stage those against a SandboxConfig pinning the new shas, and the -# mismatch would only surface at runtime as an actor wedged in STATUS_RESUMING while -# atelet rejects the download. Compare the stamp assemble.sh left against the one it -# would write now; a dir predating the stamp, or left by a failed assemble (which clears -# the stamp before overwriting anything), has no file and re-assembles. -if [[ "${need_assemble}" == "false" ]]; then - want_stamp="$(ARCH="${ARCH}" hack/microvm-assets/assemble.sh --print-stamp)" - if [[ "$(cat "${OUT}/.asset-versions" 2>/dev/null)" != "${want_stamp}" ]]; then - log "Asset set in ${OUT} is stale (version stamp mismatch); re-assembling." - need_assemble=true - fi -fi -if [[ "${need_assemble}" == "true" ]]; then - log "Assembling micro-VM assets into ${OUT} (ARCH=${ARCH})..." - ARCH="${ARCH}" OUT="${OUT}" hack/microvm-assets/assemble.sh -else - log "Assets already present in ${OUT}; skipping assemble." -fi - -# --- 2. stage assets to rustfs (kind) / GCS (GKE) -------------------------- -# Upload the four assets under kata-assets/, where atelet fetches them: the -# in-cluster rustfs (S3 API) on kind, or the GCS asset bucket on GKE. -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - log "Staging assets to in-cluster rustfs bucket ${BUCKET_NAME} (kata-assets/)..." - OUT="${OUT}" BUCKET="${BUCKET_NAME}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/microvm-assets/stage-to-rustfs.sh -else - log "Uploading assets to gs://${ASSET_BUCKET_NAME}/kata-assets/ ..." - OUT="${OUT}" BUCKET="${ASSET_BUCKET_NAME}" hack/microvm-assets/stage-to-gcs.sh -fi - -# --- 3. apply the cluster-wide microvm SandboxConfig ----------------------- -# Every asset is downloaded rather than built, so all four carry committed, -# reproducible per-arch shas and the bucket's scheme and name are the only -# substitutions left. -log "Applying microvm SandboxConfig from ${MANIFEST_TEMPLATE}..." -sed -e "s|\${BUCKET_SCHEME}|${BUCKET_SCHEME}|g" \ - -e "s|\${ASSET_BUCKET_NAME}|${ASSET_BUCKET_NAME}|g" \ - "${MANIFEST_TEMPLATE}" \ - | run_kubectl apply -f - - -log "Done. Installed cluster-wide microvm SandboxConfig." +go run ./cmd/ate-setup "${action}" microvm-deps diff --git a/hack/microvm-assets/README.md b/hack/microvm-assets/README.md index 1436ef4e68..06a3c74fc3 100644 --- a/hack/microvm-assets/README.md +++ b/hack/microvm-assets/README.md @@ -12,37 +12,34 @@ set. The asset set is four files: - `vmlinux` — the guest kernel (from kata-static) - `rootfs.img` — the guest rootfs image (from kata-static) -These helpers assemble the asset set for your node arch, stage it into the cluster's rustfs -S3 bucket, and the demo manifest's `SandboxConfig` points at it. When `/dev/kvm` is -available, `hack/create-kind-cluster.sh` mounts it into the node; atelet then advertises -it as a device, which is what places microVM workers there. +`go run ./cmd/ate-setup deploy microvm-deps` (or the `hack/install-microvm-deps.sh --install` +shim) assembles the asset set for your node arch into `bin/microvm-assets/$ARCH`, stages it +into the cluster's object store (`rustfs` on kind, GCS on GKE), and applies the cluster-wide +`microvm` `SandboxConfig`. When `/dev/kvm` is available, `hack/create-kind-cluster.sh` mounts +it into the node; atelet then advertises it as a device, which is what places microVM +workers there. > [!TIP] -> `hack/run-microvm-demo.sh` automates the full bring-up below (assets, control plane, -> demo apply) for kind OR GKE without editing committed files. The steps here are the -> manual equivalent. +> `hack/run-microvm-demo.sh` (and `hack/run-microvm-demo-kind.sh` for kind) automates the +> full bring-up below (control plane, micro-VM assets + `SandboxConfig`, and demo apply) +> without editing committed files. ## Steps (run on a KVM-capable Linux host matching the node arch) -1. **Assemble assets for your arch:** +1. **Bring up the cluster + control plane:** ```sh - ARCH=arm64 hack/microvm-assets/assemble.sh + hack/create-kind-cluster.sh # mounts /dev/kvm into the nodes + hack/install-ate-kind.sh --deploy-ate-system # control plane + rustfs (bucket: ate-snapshots) ``` -2. **Bring up the cluster + control plane:** +2. **Assemble and stage the micro-VM assets + `SandboxConfig`:** ```sh - hack/create-kind-cluster.sh # mounts /dev/kvm into the nodes - hack/install-ate-kind.sh # control plane + rustfs (bucket: ate-snapshots) + ATE_INSTALL_KIND=true hack/install-microvm-deps.sh --install ``` -3. **Stage assets into rustfs:** +3. **Apply the demo + drive it:** ```sh - OUT="$PWD/microvm-assets-arm64" hack/microvm-assets/stage-to-rustfs.sh - ``` - -4. **Apply the demo + drive it:** - ```sh - BUCKET_NAME=ate-snapshots envsubst < demos/counter/counter-microvm.yaml.tmpl | kubectl apply -f - + hack/install-ate-kind.sh --deploy-demo-counter-microvm ``` Create an actor from `counter-microvm`, hit the in-RAM counter to increment it, suspend (checkpoint), resume on a different worker pod, and confirm the count continues — proving the diff --git a/hack/microvm-assets/assemble.sh b/hack/microvm-assets/assemble.sh deleted file mode 100755 index fc1e09f095..0000000000 --- a/hack/microvm-assets/assemble.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Assemble the micro-VM (kata + cloud-hypervisor) runtime asset set that -# ateworker-microvm fetches at runtime (fetch-not-bake). Run this on a Linux -# host of the TARGET arch. -# -# Produces, under $OUT, the four assets named as the SandboxConfig expects: -# cloud-hypervisor virtiofsd vmlinux rootfs.img -# Every asset is downloaded rather than built, so all four have reproducible bytes: -# paste their sha256 sums into the manifest -# (manifests/microvm/sandboxconfig-microvm.yaml.tmpl). -# -# ateworker drives the kata-agent directly (the kata containerd shim is NOT an asset). The -# actor rootfs is overlay(virtio-fs RO lower + guest-tmpfs upper), so virtiofsd IS an -# asset. CH's restore handshake hangs against virtiofsd v1.13.3, which kata bundled up -# to and including 4.0.0; kata 4.1.0 and later bundle v1.14.0, the first release carrying the -# vhost-0.16 / vhost-user-backend-0.22 snapshot-restore fix (REPLY_ACK). So virtiofsd -# now comes out of kata-static with the kernel and rootfs instead of being sourced -# separately per arch. -# -# Env: ARCH (arm64|amd64, default arm64), KATA_VER (4.3.1), CH_VER (v53.0), -# OUT (default ./bin/microvm-assets/$ARCH, under the gitignored bin/). -# -# Always re-downloads and overwrites — there is no incremental mode. It clears -# $OUT/.asset-versions before the first write and re-stamps it with the versions that -# produced the set only once the run completes, so the stamp is present only on a dir -# assembled end-to-end by those pins. install-microvm-deps.sh uses it to decide whether -# a cached $OUT is still current. -# `--print-stamp` prints that stamp for the current env and exits without downloading. - -set -o errexit -o nounset -o pipefail - -ROOT="$(git rev-parse --show-toplevel)" - -ARCH="${ARCH:-arm64}" -KATA_VER="${KATA_VER:-4.3.1}" -CH_VER="${CH_VER:-v53.0}" -# Not env-overridable: this is whatever KATA_VER bundles. It is declared rather than -# read off the binary because --print-stamp has to answer before anything is -# downloaded, and checked against the extracted binary below so it cannot drift from -# what kata ships. -VIRTIOFSD_VER="1.14.0" -OUT="${OUT:-${ROOT}/bin/microvm-assets/$ARCH}" - -case "$ARCH" in - arm64) CH_ASSET="cloud-hypervisor-static-aarch64" ;; - amd64) CH_ASSET="cloud-hypervisor-static" ;; - *) echo "unsupported ARCH=$ARCH" >&2; exit 1 ;; -esac - -# Identifies the asset set this script produces. Cleared before the first write into -# $OUT and re-written to $OUT/$STAMP_FILE on success; install-microvm-deps.sh compares -# it against what the current checkout would build, because the filenames stay the -# same when a pin moves and an asset dir from an older checkout is otherwise -# indistinguishable from a current one. virtiofsd is stamped even though KATA_VER -# already determines it: its version is what the CH restore handshake turns on, so the -# dir should say which one it holds. -STAMP_FILE=".asset-versions" -asset_stamp() { - printf 'arch=%s\nkata=%s\ncloud-hypervisor=%s\nvirtiofsd=%s\n' \ - "$ARCH" "$KATA_VER" "$CH_VER" "$VIRTIOFSD_VER" -} - -if [ "${1:-}" = "--print-stamp" ]; then - asset_stamp - exit 0 -fi - -WORK="$(mktemp -d)" -trap 'rm -rf "$WORK"' EXIT - -mkdir -p "$OUT" -# Drop any stamp before the first overwrite into $OUT. Assets are replaced in place, -# so a run that dies partway leaves a dir mixing old and new bytes; the stamp it -# inherited describes neither. Clearing it up front means an unstamped dir is the only -# thing a failed run can leave, whatever the pins were before. -rm -f "${OUT}/${STAMP_FILE}" -cd "$WORK" - -echo ">> Downloading kata-static ${KATA_VER} (${ARCH})..." -curl -fSL -o kata-static.tar.zst \ - "https://github.com/kata-containers/kata-containers/releases/download/${KATA_VER}/kata-static-${KATA_VER}-${ARCH}.tar.zst" -mkdir -p kata -tar --zstd -xf kata-static.tar.zst -C kata -KROOT="kata/opt/kata" - -cp "$(readlink -f "${KROOT}/share/kata-containers/vmlinux.container")" "${OUT}/vmlinux" -# rootfs.img is attached over virtio-pmem and mounted with DAX (cmd/ateworker-microvm/run.go). -# The image must meet the following requirements: -# - a size that is a multiple of 2 MiB (cloud-hypervisor) -# - partition 1 aligned to the guest page size -# - a filesystem block size equal to the guest page size -# The kata-containers.img used here is built for this, and vmlinux.container uses 4 KiB pages. -# See https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder/image-builder/image_builder.sh -cp "$(readlink -f "${KROOT}/share/kata-containers/kata-containers.img")" "${OUT}/rootfs.img" -# Statically linked, so it runs as-is outside the kata layout it is packaged for. -cp "${KROOT}/libexec/virtiofsd" "${OUT}/virtiofsd" -chmod +x "${OUT}/virtiofsd" - -echo ">> Downloading cloud-hypervisor ${CH_VER} (${CH_ASSET})..." -curl -fSL -o "${OUT}/cloud-hypervisor" \ - "https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${CH_VER}/${CH_ASSET}" -chmod +x "${OUT}/cloud-hypervisor" - -echo -echo ">> Assets assembled in ${OUT}:" -cd "${OUT}" -for f in cloud-hypervisor virtiofsd vmlinux rootfs.img; do - [ -f "$f" ] || { echo "MISSING: $f" >&2; exit 1; } -done -# The stamp names a virtiofsd version, so confirm the tarball carried that one before -# writing it: a kata-side bump would otherwise stamp a version this dir does not hold. -# Only checkable where the binary runs, and assembling for another arch (or on macOS) -# is legitimate, so a binary this host cannot exec is skipped rather than fatal. -if GOT_VIRTIOFSD="$("${OUT}/virtiofsd" --version 2>/dev/null | head -1 | awk '{print $2}')" \ - && [ -n "${GOT_VIRTIOFSD}" ]; then - if [ "${GOT_VIRTIOFSD}" != "${VIRTIOFSD_VER}" ]; then - echo "kata ${KATA_VER} bundles virtiofsd ${GOT_VIRTIOFSD}, not ${VIRTIOFSD_VER}: update VIRTIOFSD_VER" >&2 - exit 1 - fi - echo "virtiofsd ${GOT_VIRTIOFSD}" -fi -# Written only once all four are present and virtiofsd matches, and only after the -# up-front rm, so the stamp exists exactly when this dir was assembled end-to-end by -# these pins. -asset_stamp > "${OUT}/${STAMP_FILE}" -echo -echo ">> sha256 (paste all four into the per-arch block in" -echo ">> manifests/microvm/sandboxconfig-microvm.yaml.tmpl):" -sha256sum cloud-hypervisor virtiofsd vmlinux rootfs.img diff --git a/hack/microvm-assets/stage-to-gcs.sh b/hack/microvm-assets/stage-to-gcs.sh deleted file mode 100755 index 7820e6c9c3..0000000000 --- a/hack/microvm-assets/stage-to-gcs.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Stage the assembled micro-VM asset set into the GCS asset bucket under -# kata-assets/, where atelet fetches it through the node's object-store plugin -# (per manifests/microvm/sandboxconfig-microvm.yaml.tmpl). The plugin refuses -# assets stored under a snapshot location (/atespaces/), -# so keep them outside it. The GKE counterpart of stage-to-rustfs.sh. Run after -# assemble.sh has produced $OUT. -# -# Requires the `gcloud` CLI authenticated for the bucket's project. Env: OUT (asset -# dir, default ./bin/microvm-assets/amd64), BUCKET (required, the asset bucket, -# BUCKET_NAME-assets as install-microvm-deps.sh passes it), PROJECT_ID (optional; -# passed to gcloud as --project when set). - -set -o errexit -o nounset -o pipefail - -ROOT="$(git rev-parse --show-toplevel)" - -OUT="${OUT:-${ROOT}/bin/microvm-assets/amd64}" -BUCKET="${BUCKET:?set BUCKET to the asset bucket, BUCKET_NAME-assets}" - -# Pass --project only when PROJECT_ID is set (mirrors hack/teardown.sh); otherwise -# gcloud uses its active config project. ${PROJECT_ID:+...} elides the flag entirely -# when unset (same idiom as KUBECTL_CONTEXT in hack/run-microvm-demo.sh). -echo ">> Uploading assets to gs://${BUCKET}/kata-assets/ ..." -for f in cloud-hypervisor virtiofsd vmlinux rootfs.img; do - echo " $f" - gcloud storage cp ${PROJECT_ID:+--project="${PROJECT_ID}"} "${OUT}/${f}" "gs://${BUCKET}/kata-assets/${f}" -done - -echo ">> Done. Verify:" -gcloud storage ls ${PROJECT_ID:+--project="${PROJECT_ID}"} "gs://${BUCKET}/kata-assets/" diff --git a/hack/microvm-assets/stage-to-rustfs.sh b/hack/microvm-assets/stage-to-rustfs.sh deleted file mode 100755 index 5fbd9c1e8b..0000000000 --- a/hack/microvm-assets/stage-to-rustfs.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Stage the assembled micro-VM asset set into the kind cluster's rustfs S3 bucket -# under kata-assets/, where atelet fetches it through the node's object-store -# plugin (per manifests/microvm/sandboxconfig-microvm.yaml.tmpl). The plugin -# refuses assets stored under a snapshot location -# (/atespaces/), so keep them outside it. -# Run after the cluster is up (hack/install-ate-kind.sh) and assemble.sh has produced $OUT. -# -# The S3 client runs in a throwaway container (the same pinned amazon/aws-cli -# image as the rustfs-bucket-init Job), so developers don't need the `aws` CLI -# installed. The container joins the kind node's network namespace, which is what -# makes rustfs's ClusterIP routable: a container merely attached to the `kind` -# docker network can reach the node's own IP but has no route to the service or -# pod CIDRs. That also avoids a `kubectl port-forward`, which silently uploads -# nothing when it targets the wrong cluster, and which a container can't reach at -# all on macOS (the container's localhost is the Docker VM, not the host). -# -# Env: OUT (asset dir, default ./bin/microvm-assets/arm64), BUCKET (default -# ate-snapshots), NAMESPACE (rustfs namespace, default ate-system), -# KUBECTL_CONTEXT (optional; kube context), KIND_CLUSTER_NAME (default: derived -# from KUBECTL_CONTEXT, else "kind"). - -set -o errexit -o nounset -o pipefail - -ROOT="$(git rev-parse --show-toplevel)" - -OUT="${OUT:-${ROOT}/bin/microvm-assets/arm64}" -BUCKET="${BUCKET:-ate-snapshots}" -NAMESPACE="${NAMESPACE:-ate-system}" -KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" -# kind contexts are named kind-; fall back to kind's own default. -KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-${KUBECTL_CONTEXT#kind-}}" -KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" - -# Keep in sync with the rustfs-bucket-init Job in -# manifests/ate-install/kind/rustfs.yaml, which creates the bucket we upload into. -AWS_CLI_IMAGE="amazon/aws-cli:2.38.5@sha256:c8156a0e74e13929a42d7e60f0d4e14e7d28d980cd21822505d4bff80e5460c1" - -ASSETS=(cloud-hypervisor virtiofsd vmlinux rootfs.img) - -run_kubectl() { - kubectl ${KUBECTL_CONTEXT:+--context="${KUBECTL_CONTEXT}"} -n "${NAMESPACE}" "$@" -} - -if ! command -v docker >/dev/null 2>&1; then - echo "error: 'docker' is required to run the S3 client but was not found in PATH" >&2 - exit 1 -fi - -for f in "${ASSETS[@]}"; do - if [[ ! -f "${OUT}/${f}" ]]; then - echo "error: missing asset ${OUT}/${f}; run hack/microvm-assets/assemble.sh first" >&2 - exit 1 - fi -done - -# rustfs must be serving and the bucket must exist before anything is uploaded. -echo ">> Waiting for rustfs in namespace ${NAMESPACE}..." -run_kubectl rollout status deploy/rustfs --timeout=300s -run_kubectl wait --for=condition=Complete job/rustfs-bucket-init --timeout=300s - -NODE="$("${ROOT}/hack/kind.sh" get nodes --name "${KIND_CLUSTER_NAME}" | head -n1)" -if [[ -z "${NODE}" ]]; then - echo "error: no nodes found for kind cluster '${KIND_CLUSTER_NAME}'" >&2 - exit 1 -fi -ENDPOINT="http://$(run_kubectl get svc rustfs -o jsonpath='{.spec.clusterIP}'):9000" - -echo ">> Uploading assets to s3://${BUCKET}/kata-assets/ via ${ENDPOINT} (netns of ${NODE})..." -aws_cli() { - # -i (no -t): a TTY would corrupt the binary stream piped in on stdin. - docker run --rm -i \ - --network "container:${NODE}" \ - -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-rustfsadmin}" \ - -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-rustfsadmin}" \ - -e AWS_REGION="${AWS_REGION:-us-east-1}" \ - -e AWS_ENDPOINT_URL="${ENDPOINT}" \ - "${AWS_CLI_IMAGE}" "$@" -} - -for f in "${ASSETS[@]}"; do - echo " $f" - aws_cli s3 cp - "s3://${BUCKET}/kata-assets/${f}" < "${OUT}/${f}" -done - -echo ">> Done. Verify:" -aws_cli s3 ls "s3://${BUCKET}/kata-assets/" < /dev/null diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 995cb028f6..7a7bc83d2d 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -19,9 +19,9 @@ # and calls this script), mirroring install-ate.sh / install-ate-kind.sh. # # Composes: -# 1. hack/install-ate.sh --deploy-ate-system (control plane) -# 2. hack/install-microvm-deps.sh --install (asset build/stage + cluster-wide -# microvm SandboxConfig) +# 1. hack/install-ate.sh --deploy-ate-system (control plane) +# 2. go run ./cmd/ate-setup deploy microvm-deps (asset build/stage + cluster-wide +# microvm SandboxConfig) # 3. Deploy the counter-microvm demo (worker pool manifest, atespace, and # ActorTemplate through the ate API). # @@ -90,10 +90,10 @@ else fi # --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- -# install-microvm-deps.sh handles the assemble/stage/apply flow. Ordering +# `ate-setup deploy microvm-deps` handles the assemble/stage/apply flow. Ordering # matters: the control plane must be up so the SandboxConfig CRD exists. log "Installing micro-VM dependencies..." -KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-microvm-deps.sh --install +KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" go run ./cmd/ate-setup deploy microvm-deps # --- 3. apply the demo ------------------------------------------------------ KCTX_FLAG="" diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index 19bd5b6fd3..3a0f252440 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -15,10 +15,11 @@ # Cluster-wide SandboxConfig for the micro-VM (kata + cloud-hypervisor) sandbox # class. Unlike sandboxconfig-gvisor.yaml (applied unconditionally by # hack/install-ate.sh --deploy-ate-system), this is opt-in: apply via -# hack/install-microvm-deps.sh --install after staging the asset set -# (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh). A microvm WorkerPool -# resolves it as its class default (or names it via configRef: microvm), so -# a missing/stale config shows on the pool's status. +# hack/install-microvm-deps.sh --install (or `go run ./cmd/ate-setup deploy +# microvm-deps`), which assembles the asset set, stages it to the cluster's +# object store, and applies this manifest. A microvm WorkerPool resolves it as +# its class default (or names it via configRef: microvm), so a missing/stale +# config shows on the pool's status. # # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs) # are FETCHED at runtime from the asset bucket ${ASSET_BUCKET_NAME} under @@ -30,10 +31,10 @@ # boots cloud-hypervisor itself and gives the actor an overlay rootfs (virtio-fs # RO lower + guest-tmpfs upper), so it fetches virtiofsd too. # -# The per-arch sha256 values below are the asset sets produced by -# hack/microvm-assets/assemble.sh; atelet selects the block matching the node's -# architecture, and each cluster's bucket holds that arch's binaries at these -# paths (staged by hack/install-microvm-deps.sh --install). +# The per-arch sha256 values below are the asset sets assembled by +# cmd/ate-setup/internal/steps/microvm.go; atelet selects the block matching +# the node's architecture, and each cluster's bucket holds that arch's binaries +# at these paths (staged by hack/install-microvm-deps.sh --install). apiVersion: ate.dev/v1 kind: SandboxConfig @@ -62,7 +63,7 @@ spec: kata-image: url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/rootfs.img" sha256: "69f9a76e5a2563b7c74d1f271f3a904240f50ef405465f832021525894519f97" - # amd64 assets are kata 4.3.1 (assemble.sh ARCH=amd64). + # amd64 assets are kata 4.3.1 (ARCH=amd64). amd64: cloud-hypervisor: url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/cloud-hypervisor" From d5baa467b78438f42a7df0ed06bd57b9dc00273d Mon Sep 17 00:00:00 2001 From: Noah West Date: Fri, 9 Oct 2026 17:47:45 -0700 Subject: [PATCH 03/10] ate-setup: fold microvm-deps into deploy and delete ate-system Stage the micro-VM guest assets and apply the microvm SandboxConfig as part of deploy ate-system (and deploy sandboxconfig), with a --skip-microvm-deps opt-out flag, and remove the microvm SandboxConfig during delete ate-system. Drop the standalone deploy/delete microvm-deps subcommands and the hack/install-microvm-deps.sh shim. Signed-off-by: Noah West --- benchmarking/automation/README.md | 11 ++- benchmarking/automation/orchestrator.py | 26 +----- benchmarking/deploy_locust.sh | 3 +- benchmarking/workloads/deploy.sh | 3 +- cmd/ate-setup/commands.md | 29 +++--- cmd/ate-setup/differences.md | 14 +-- cmd/ate-setup/internal/cmd/benchmarks.go | 6 +- cmd/ate-setup/internal/cmd/deploy.go | 41 ++++++--- cmd/ate-setup/internal/cmd/deploy_test.go | 59 +++++++++++- cmd/ate-setup/internal/cmd/microvm.go | 60 ------------- cmd/ate-setup/internal/cmd/microvm_test.go | 50 ----------- .../demos/countermicrovm/countermicrovm.go | 6 +- .../demos/egressmicrovm/egressmicrovm.go | 12 ++- cmd/ate-setup/internal/steps/benchmarks.go | 30 ++----- cmd/ate-setup/internal/steps/delete.go | 31 ++----- cmd/ate-setup/internal/steps/delete_test.go | 89 ------------------- cmd/ate-setup/internal/steps/deploy.go | 31 ++++--- cmd/ate-setup/internal/steps/microvm_test.go | 44 --------- demos/counter/README.md | 11 ++- demos/counter/counter-microvm.yaml.tmpl | 3 +- demos/egress/egress-microvm.yaml.tmpl | 3 +- docs/dev/microvm-local.md | 11 ++- docs/dev/mitm-egress-test-runbook.md | 7 -- docs/operator-install.md | 3 +- hack/install-microvm-deps.sh | 63 ------------- hack/microvm-assets/README.md | 54 ----------- hack/run-microvm-demo.sh | 31 +++---- .../e2e/suites/egressmitm/egressmitm_test.go | 3 - .../microvm/sandboxconfig-microvm.yaml.tmpl | 15 ++-- tools/setup-gcp/README.md | 2 +- tools/setup-gcp/cmd/common.go | 2 +- 31 files changed, 187 insertions(+), 566 deletions(-) delete mode 100644 cmd/ate-setup/internal/cmd/microvm.go delete mode 100644 cmd/ate-setup/internal/cmd/microvm_test.go delete mode 100755 hack/install-microvm-deps.sh delete mode 100644 hack/microvm-assets/README.md diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index de34c358b5..27f73f65ec 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -20,17 +20,16 @@ the router capacity benchmark — see and/or `${KO_DOCKER_REPO}/nighthawk-ingress-test:`. 5. `hack/install-ate.sh deploy ate-system` (with `ATE_CREDENTIAL_PROVIDER` in `.ate-dev-env.sh`, or `--credential-provider` in the test's - `ateArgs`) + `benchmarking/workloads/deploy.sh + `ateArgs`, which also stages the micro-VM guest assets to the cluster's + object store bucket and applies both the `gvisor-default` and `microvm` + `SandboxConfig`s) + `benchmarking/workloads/deploy.sh --deploy --sandbox-class ` (these build & push substrate / workload images via `ko` as part of their deploy steps — there's no separate image build step). The install sets `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` to the `ate-client` and `benchmarking/benchmark-runner` ServiceAccounts so the runner's calls pass [authorization](../../docs/authorization.md), and sets `ATE_INGRESS_ALLOWED_SPIFFE_IDS` to `benchmarking/benchmark-runner` for the - Envoy ingress router. For a `microvm` test the orchestrator also - runs `go run ./cmd/ate-setup deploy microvm-deps` between the two, which - stages kata + cloud-hypervisor + virtiofsd assets to the cluster's object - store bucket and applies the cluster-wide `microvm` SandboxConfig. + Envoy ingress router. For a `nighthawk-ingress` test the orchestrator additionally patches the `atenet-router` Deployment right after `deploy_substrate`: `envoyCpu` is the benchmark's independent variable and the shipped manifest sets @@ -47,7 +46,7 @@ the router capacity benchmark — see variable (a JSON object) if set. A `--metadata` in the test's own `flags` is merged in too, its keys winning on conflict. - Polls until complete/failed/timeout; tails logs; deletes the Job. - - Tears down workloads + microVM deps (if any) + substrate. + - Tears down workloads + substrate. - If not the last test, redeploys them so the next run starts clean. ## Choosing a sandbox class diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index fd72c59ebe..2eb06df4e5 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -420,20 +420,6 @@ def teardown_substrate() -> None: run_no_check(["hack/install-ate.sh", "delete", "ate-system"]) -def install_microvm_deps() -> None: - """Stage kata/cloud-hypervisor assets and apply the cluster-wide - microvm SandboxConfig. Required before a microvm WorkerPool can run - actors; must run after deploy_substrate() (which installs the CRDs).""" - run(["go", "run", "./cmd/ate-setup", "deploy", "microvm-deps"]) - - -def teardown_microvm_deps() -> None: - """Remove the microvm SandboxConfig. Must run before - teardown_substrate(), which deletes the SandboxConfig CRD (and would - prevent this from succeeding via kubectl).""" - run_no_check(["go", "run", "./cmd/ate-setup", "delete", "microvm-deps"]) - - def deploy_workloads( worker_count: int = 1, sandbox_class: str = "gvisor", @@ -632,11 +618,8 @@ def main() -> None: # fire that crashed mid-test (or any other process that left # state behind) would otherwise leak its substrate + workloads # into this run. All teardowns use --ignore-not-found, so - # this is cheap on a clean cluster. Order matters: - # microvm-deps deletes a SandboxConfig CR, which requires the - # SandboxConfig CRD that teardown_substrate removes. + # this is cheap on a clean cluster. teardown_workloads() - teardown_microvm_deps() teardown_substrate() sandbox_class = test.get("sandboxClass", "gvisor") @@ -646,10 +629,6 @@ def main() -> None: try: deploy_substrate(substrate_args(test)) TYPES[ttype].pre_test(test) - # install-microvm-deps needs the CRDs from deploy_substrate; - # deploy_workloads needs the microvm SandboxConfig. - if sandbox_class == "microvm": - install_microvm_deps() deploy_workloads( test.get("workerCount", 1), sandbox_class, @@ -678,10 +657,7 @@ def main() -> None: finally: # Always tear down, even if deploy or run failed, so the # next test (and the next CronJob fire) starts clean. - # microvm-deps must go before substrate for the same reason - # as above. teardown_workloads() - teardown_microvm_deps() teardown_substrate() duration = time.time() - start_time results.append((test["name"], status, duration, failure_msg)) diff --git a/benchmarking/deploy_locust.sh b/benchmarking/deploy_locust.sh index 6d216125cd..1dfdc96567 100755 --- a/benchmarking/deploy_locust.sh +++ b/benchmarking/deploy_locust.sh @@ -45,8 +45,7 @@ usage() { echo " --deploy Deploy workloads, build/push locust image, then deploy locust" echo " --delete Delete locust and then workloads" echo " --worker-count N Number of WorkerPool replicas (default: 1)" - echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)." - echo " microvm requires hack/install-microvm-deps.sh --install to have run." + echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)" echo " --otlp-endpoint URL Forwarded to workloads/deploy.sh. The address to which an" echo " instrumented actor container sends telemetry." echo " --actor-memory SIZE Forwarded to workloads/deploy.sh. Memory limit for the" diff --git a/benchmarking/workloads/deploy.sh b/benchmarking/workloads/deploy.sh index 010a2cc295..fccb9bc56e 100755 --- a/benchmarking/workloads/deploy.sh +++ b/benchmarking/workloads/deploy.sh @@ -93,8 +93,7 @@ usage() { echo " --deploy Substitute env vars and deploy workloads to the cluster using ko apply" echo " --delete Substitute env vars and delete workloads from the cluster" echo " --worker-count N Number of WorkerPool replicas (default: 1)" - echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)." - echo " microvm requires hack/install-microvm-deps.sh --install to have run." + echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)" echo " --actor-memory SIZE Memory limit for the benchmark ActorTemplates (default: 256Mi)" echo " --actor-cpu CPU CPU limit for the benchmark ActorTemplates (default: 1)" echo " --worker-memory SIZE Memory request and limit for each WorkerPool pod" diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index a42f686cde..d7f58e544a 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -109,25 +109,22 @@ that already names a manifest is used as written, and is not looked up. |---|---| | `deploy ate-system` | `--deploy-ate-system` | | `deploy ate-system --setup-csi=nfs` | `--deploy-ate-system --setup-csi=nfs` | +| `deploy ate-system --skip-microvm-deps` | `--deploy-ate-system --skip-microvm-deps` | | `deploy atelet` | `--deploy-atelet` | | `deploy apiserver` | `--deploy-ate-apiserver` | | `deploy ate-controller` | (no shell equivalent) | | `deploy atenet` | `--deploy-atenet` | | `deploy podcertificate-controller` | (no shell equivalent) | -| `deploy sandboxconfig` | (no shell equivalent) | +| `deploy sandboxconfig [--skip-microvm-deps]` | (no shell equivalent) | | `deploy postgres` | (no shell equivalent) | -| `deploy microvm-deps` | `hack/install-microvm-deps.sh --install` | `deploy ate-system` is the whole control plane: CRDs, RBAC, the store, the -apiserver, the controller, atenet, and atelet. It creates every `create` -resource below on the way, so those subcommands are only needed to redo one on -a running cluster. - -`deploy microvm-deps` makes the cluster able to run micro-VM workers: it -assembles the guest assets, stages them in the object store the atelet pulls -from, and applies the cluster-wide `microvm` SandboxConfig. `deploy ate-system` -installs only `gvisor-default`, so this is needed before any micro-VM demo or -WorkerPool. `deploy benchmarks --sandbox-class=microvm` runs it for you. +apiserver, the controller, atenet, and atelet. Unless `--skip-microvm-deps` is +passed (also accepted by `deploy sandboxconfig`), it also stages the micro-VM +guest assets into the cluster object store and applies the `microvm` +cluster-wide `SandboxConfig` alongside `gvisor-default`. It creates every +`create` resource below on the way, so those subcommands are only needed to +redo one on a running cluster. ## Publish @@ -158,15 +155,9 @@ attaches the files to the draft release. | `delete ate-system` | `--delete-ate-system` | | `delete atenet` | `--delete-atenet` | | `delete all` | `--delete-all` | -| `delete microvm-deps` | `hack/install-microvm-deps.sh --delete` | `delete all` removes every registered demo and then the control plane. -`delete microvm-deps` removes only the cluster-wide `microvm` SandboxConfig; the -staged assets stay in the object store, which outlives any one install. It is -not part of `delete all`, since the SandboxConfig may belong to something other -than the install being removed. - ## Create Individual secrets and config that `deploy ate-system` creates automatically. @@ -223,9 +214,9 @@ See |---|---|---| | `deploy demo counter` | `--deploy-demo-counter` | A counter actor exercising snapshot, resume, and atenet ingress | | `deploy demo counter --with-external-volume [--storage-class NAME]` | `--deploy-demo-counter-with-external-volume` (`STORAGE_CLASS=NAME`) | The same, plus an external volume and a pre-seeded file to validate (requires `--preview=ExternalVolumes`). Run `setup csi` first and name the class it created, e.g. `csi-nfs-sc`; defaults to `standard` | -| `deploy demo counter-microvm` | `--deploy-demo-counter-microvm` | The counter demo on microVM workers. Run `deploy microvm-deps` first | +| `deploy demo counter-microvm` | `--deploy-demo-counter-microvm` | The counter demo on microVM workers | | `deploy demo egress` | `--deploy-demo-egress` | Egress policy enforcement through atenet; the actors trust the egress gateway's CA | -| `deploy demo egress-microvm` | `--deploy-demo-egress-microvm` | The same on microVM workers. Run `deploy microvm-deps` first | +| `deploy demo egress-microvm` | `--deploy-demo-egress-microvm` | The same on microVM workers | | `deploy demo jupyter` | `--deploy-demo-jupyter` | A Jupyter notebook server per actor, reached through atenet ingress | | `deploy demo sandbox` | `--deploy-demo-sandbox` | An on-demand sandbox actor driven by the sandbox client | | `deploy demo multi-template` | `--deploy-demo-multi-template` | Two ActorTemplates sharing one WorkerPool | diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 12b0af9dfb..cfbdd12268 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -1,11 +1,12 @@ # ate-setup vs. the install shell scripts `ate-setup` is a Go port of `hack/install-ate.sh`, the seven -`hack/install-demo-*.sh` scripts it sourced, and `hack/setup-csi-*-kind.sh`. +`hack/install-demo-*.sh` scripts it sourced, `hack/setup-csi-*-kind.sh`, and +`hack/install-microvm-deps.sh`. -The port has replaced them. The demo scripts are deleted, and -`hack/install-ate.sh` is a shim holding no install logic of its own: it -translates the flags and environment variables the installer has always +The port has replaced them. The demo, CSI, and micro-VM asset scripts are +deleted, and `hack/install-ate.sh` is a shim holding no install logic of its +own: it translates the flags and environment variables the installer has always accepted onto `ate-setup` commands, so existing command lines keep working. See [`commands.md`](commands.md) for the flag-by-flag mapping. @@ -97,8 +98,9 @@ is fronted by `deploy`/`delete benchmarks`. The micro-VM asset assembly (`kata-static` `.tar.zst` extraction and `cloud-hypervisor` download), object-store staging (`rustfs` on Kind, GCS on -GKE), and `microvm` `SandboxConfig` lifecycle live in `deploy`/`delete -microvm-deps`, with `hack/install-microvm-deps.sh` kept as a translation shim. +GKE), and `microvm` `SandboxConfig` lifecycle now run in Go as part of `deploy +ate-system` (and `deploy sandboxconfig`), with `delete ate-system` removing the +`microvm` `SandboxConfig` alongside the rest of the control plane. `ko` is no longer asked to apply anything. The scripts ran `run_ko apply`, which made ko shell out to kubectl and forced the awkward `-- --context=` special case diff --git a/cmd/ate-setup/internal/cmd/benchmarks.go b/cmd/ate-setup/internal/cmd/benchmarks.go index d92d1c4b6e..670cf9b495 100644 --- a/cmd/ate-setup/internal/cmd/benchmarks.go +++ b/cmd/ate-setup/internal/cmd/benchmarks.go @@ -82,11 +82,7 @@ See benchmarking/README.md for the walkthrough and customization options.`, var deleteBenchmarksCmd = &cobra.Command{ Use: "benchmarks", Short: "Delete the locust load test stack and the benchmark workloads", - Long: `Delete the locust load test stack and the benchmark workloads. - -Pass --sandbox-class=microvm to also remove the micro-VM SandboxConfig; it is -cluster-wide, so it is left in place by default.`, - Args: cobra.NoArgs, + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { opts, err := benchmarkOptions() if err != nil { diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 76d3f12ff0..da01aff4ba 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -27,8 +27,11 @@ var deployCmd = &cobra.Command{ Short: "Deploy Agent Substrate components", } -// deployAteSystemPath is the command path the ate-system settings bind to. -const deployAteSystemPath = "deploy ate-system" +// Command paths the deploy subcommands bind their flags to. +const ( + deployAteSystemPath = "deploy ate-system" + deploySandboxConfigPath = "deploy sandboxconfig" +) // deployOptions reads the settings deploy ate-system owns. // @@ -38,7 +41,10 @@ const deployAteSystemPath = "deploy ate-system" // correct for an argument: there is no channel to name. func deployOptions() (steps.DeployOptions, error) { r := env.Cfg.Resolved() - opts := steps.DeployOptions{SetupCSI: r.String("csi.setup")} + opts := steps.DeployOptions{ + SetupCSI: r.String("csi.setup"), + SkipMicroVMDeps: r.Bool("skipMicroVMDeps"), + } if err := steps.ValidateCSIDriver(opts.SetupCSI); err != nil { v, _ := r.Value("csi.setup") return opts, &config.InvalidError{Value: v, Want: "nfs, hostpath, both or none"} @@ -53,7 +59,9 @@ var deployAteSystemCmd = &cobra.Command{ This installs the CRDs and RBAC, the podcertificate controller and the secrets it signs, PostgreSQL, ate-api-server, ate-idp-server, ate-controller, the atenet -dataplane, and the atelet DaemonSet, then waits for each to roll out. +dataplane, the default gVisor and micro-VM SandboxConfigs (staging the micro-VM +guest assets to the cluster object store), and the atelet DaemonSet, then waits +for each to roll out. The bundled PostgreSQL StatefulSet is skipped when ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING or the ATE_API_POSTGRES_CLOUDSQL_* variables @@ -138,10 +146,10 @@ var deployPodCertControllerCmd = &cobra.Command{ var deploySandboxConfigCmd = &cobra.Command{ Use: "sandboxconfig", - Short: "Deploy the SandboxConfig admission policy and the default gVisor SandboxConfig only", + Short: "Deploy the SandboxConfig admission policy and the default gVisor and micro-VM SandboxConfigs only", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - return env.DeploySandboxConfig(cmd.Context()) + return env.DeploySandboxConfig(cmd.Context(), env.Cfg.Resolved().Bool("skipMicroVMDeps")) }, } @@ -173,12 +181,21 @@ func init() { deployPostgresCmd, ) - config.RegisterCommand(config.Setting{ - Key: "csi.setup", Env: "SETUP_CSI", Flag: "setup-csi", - Kind: config.KindString, Default: "none", - Commands: []string{deployAteSystemPath}, - Usage: "Also install CSI driver (nfs, hostpath, both, none; default: none)", - }) + config.RegisterCommand( + config.Setting{ + Key: "csi.setup", Env: "SETUP_CSI", Flag: "setup-csi", + Kind: config.KindString, Default: "none", + Commands: []string{deployAteSystemPath}, + Usage: "Also install CSI driver (nfs, hostpath, both, none; default: none)", + }, + config.Setting{ + Key: "skipMicroVMDeps", Env: "ATE_INSTALL_SKIP_MICROVM_DEPS", Flag: "skip-microvm-deps", + Kind: config.KindBool, Default: "false", + Commands: []string{deployAteSystemPath, deploySandboxConfigPath}, + Usage: "Skip staging micro-VM guest assets and applying the microvm SandboxConfig", + }, + ) config.BindCommandFlags(deployAteSystemPath, deployAteSystemCmd.Flags()) + config.BindCommandFlags(deploySandboxConfigPath, deploySandboxConfigCmd.Flags()) deployAteSystemCmd.Flags().Lookup("setup-csi").NoOptDefVal = "none" } diff --git a/cmd/ate-setup/internal/cmd/deploy_test.go b/cmd/ate-setup/internal/cmd/deploy_test.go index 31f91963db..eb1e1dc72f 100644 --- a/cmd/ate-setup/internal/cmd/deploy_test.go +++ b/cmd/ate-setup/internal/cmd/deploy_test.go @@ -14,7 +14,12 @@ package cmd -import "testing" +import ( + "testing" + + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" +) // An upgrade or a rollback moves the control plane one component at a time, // so the components that only deploy ate-system used to install have their own @@ -42,3 +47,55 @@ func TestComponentDeploySubcommands(t *testing.T) { }) } } + +func TestDeploySkipMicroVMDepsFlag(t *testing.T) { + t.Run("deploy ate-system", func(t *testing.T) { + f := deployAteSystemCmd.Flags().Lookup("skip-microvm-deps") + t.Cleanup(func() { + _ = f.Value.Set("false") + f.Changed = false + }) + if err := deployAteSystemCmd.ParseFlags([]string{"--skip-microvm-deps"}); err != nil { + t.Fatalf("ParseFlags(--skip-microvm-deps): %v", err) + } + r, err := config.Resolve(deployAteSystemCmd.Flags(), config.ResolveOptions{}) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + cfg := &config.Config{} + cfg.SetResolved(r) + + prev := env + env = &steps.Env{Cfg: cfg} + t.Cleanup(func() { env = prev }) + + opts, err := deployOptions() + if err != nil { + t.Fatalf("deployOptions(): %v", err) + } + if !opts.SkipMicroVMDeps { + t.Error("opts.SkipMicroVMDeps = false, want true") + } + }) + + t.Run("deploy sandboxconfig", func(t *testing.T) { + f := deploySandboxConfigCmd.Flags().Lookup("skip-microvm-deps") + if f == nil { + t.Fatal("deploy sandboxconfig is missing --skip-microvm-deps flag") + } + t.Cleanup(func() { + _ = f.Value.Set("false") + f.Changed = false + }) + if err := deploySandboxConfigCmd.ParseFlags([]string{"--skip-microvm-deps"}); err != nil { + t.Fatalf("ParseFlags(--skip-microvm-deps): %v", err) + } + r, err := config.Resolve(deploySandboxConfigCmd.Flags(), config.ResolveOptions{}) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if !r.Bool("skipMicroVMDeps") { + t.Error("r.Bool(\"skipMicroVMDeps\") = false, want true") + } + }) +} diff --git a/cmd/ate-setup/internal/cmd/microvm.go b/cmd/ate-setup/internal/cmd/microvm.go deleted file mode 100644 index 97a31140e3..0000000000 --- a/cmd/ate-setup/internal/cmd/microvm.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "github.com/spf13/cobra" -) - -var deployMicroVMDepsCmd = &cobra.Command{ - Use: "microvm-deps", - Short: "Deploy the micro-VM prerequisites: guest assets and the microvm SandboxConfig", - Long: `Make the cluster able to run micro-VM workers. - -This assembles the micro-VM guest assets, stages them in the object store the -atelet pulls from, and applies the cluster-wide "microvm" SandboxConfig. - -"deploy ate-system" installs only the gvisor-default SandboxConfig, so run this -before anything that asks for micro-VM workers: "deploy demo counter-microvm", -"deploy demo egress-microvm", or a WorkerPool of your own. -"deploy benchmarks --sandbox-class=microvm" does it for you. - -Re-running is cheap: assets whose recorded versions still match are left alone.`, - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, _ []string) error { - return env.DeployMicroVMDeps(cmd.Context()) - }, -} - -var deleteMicroVMDepsCmd = &cobra.Command{ - Use: "microvm-deps", - Short: "Delete the cluster-wide microvm SandboxConfig", - Long: `Remove the cluster-wide "microvm" SandboxConfig. - -The staged assets are left in the object store: the bucket outlives any one -install, and staging them again is the slow part of "deploy microvm-deps". - -The SandboxConfig is cluster-wide, so check that nothing else is still asking -for micro-VM workers before removing it.`, - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, _ []string) error { - return env.DeleteMicroVMDeps(cmd.Context()) - }, -} - -func init() { - deployCmd.AddCommand(deployMicroVMDepsCmd) - deleteCmd.AddCommand(deleteMicroVMDepsCmd) -} diff --git a/cmd/ate-setup/internal/cmd/microvm_test.go b/cmd/ate-setup/internal/cmd/microvm_test.go deleted file mode 100644 index d675329834..0000000000 --- a/cmd/ate-setup/internal/cmd/microvm_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "testing" - - "github.com/spf13/cobra" -) - -// The two commands share a name under different parents, which cobra allows -// and a mistake in init() would silently turn into one command registered -// twice under the same parent. -func TestMicroVMDepsIsRegisteredUnderBothVerbs(t *testing.T) { - for _, tc := range []struct { - parent *cobra.Command - want *cobra.Command - }{ - {deployCmd, deployMicroVMDepsCmd}, - {deleteCmd, deleteMicroVMDepsCmd}, - } { - t.Run(tc.parent.Name(), func(t *testing.T) { - found, _, err := tc.parent.Find([]string{"microvm-deps"}) - if err != nil { - t.Fatalf("%s microvm-deps: %v", tc.parent.Name(), err) - } - if found != tc.want { - t.Fatalf("%s microvm-deps resolved to %v, want the %s subcommand", - tc.parent.Name(), found.CommandPath(), tc.parent.Name()) - } - // Staging assets and deleting a cluster-wide SandboxConfig are - // both worth refusing on a typo rather than guessing at. - if err := found.Args(found, []string{"stray"}); err == nil { - t.Error("Args([stray]) = nil, want an error") - } - }) - } -} diff --git a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go index 5b94a1b8c0..812c6836fb 100644 --- a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go +++ b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go @@ -14,9 +14,7 @@ // Package countermicrovm installs the micro-VM variant of the counter demo. // -// It needs the cluster-wide `microvm` SandboxConfig that `deploy -// microvm-deps` creates. It has no external-volume option: the CSI path is -// covered by demo-counter. +// It has no external-volume option: the CSI path is covered by demo-counter. package countermicrovm import ( @@ -32,7 +30,7 @@ const namespace = "ate-demo-counter-microvm" func init() { demos.Register(&demos.Substrate{ DemoName: "demo-counter-microvm", - Short: "The counter demo on micro-VM workers (needs `deploy microvm-deps`)", + Short: "The counter demo on micro-VM workers", WorkerPoolManifest: "demos/counter/counter-microvm.yaml.tmpl", Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "counter-microvm"}}, Templates: []demos.SubstrateTemplate{{ diff --git a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go index 613d3729d7..9047fdef98 100644 --- a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go +++ b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go @@ -14,12 +14,10 @@ // Package egressmicrovm installs the micro-VM variant of the egress demo. // -// It needs the cluster-wide `microvm` SandboxConfig that `deploy -// microvm-deps` creates. It is a demo of its own -// rather than a flag on demo-egress so that it appears in help and in the -// `delete all` sweep, and so that both can be installed side by side: the -// networking e2e suite runs against whichever the sandbox class under test -// selects. +// It is a demo of its own rather than a flag on demo-egress so that it appears +// in help and in the `delete all` sweep, and so that both can be installed side +// by side: the networking e2e suite runs against whichever the sandbox class +// under test selects. package egressmicrovm import ( @@ -35,7 +33,7 @@ const namespace = "ate-demo-egress-microvm" func init() { demos.Register(&demos.Substrate{ DemoName: "demo-egress-microvm", - Short: "Egress policy enforcement on micro-VM workers (needs `deploy microvm-deps`)", + Short: "Egress policy enforcement on micro-VM workers", WorkerPoolManifest: "demos/egress/egress-microvm.yaml.tmpl", Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "egress-microvm"}}, Templates: []demos.SubstrateTemplate{{ diff --git a/cmd/ate-setup/internal/steps/benchmarks.go b/cmd/ate-setup/internal/steps/benchmarks.go index 5087d8a834..8025488eb1 100644 --- a/cmd/ate-setup/internal/steps/benchmarks.go +++ b/cmd/ate-setup/internal/steps/benchmarks.go @@ -25,10 +25,10 @@ import ( "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) -// The benchmark and micro-VM stacks are still driven by shell. They orchestrate -// several other scripts (image builds, asset assembly, object-store staging) -// that are out of scope for this command, so ate-setup shells out to them with -// a translated argument list rather than reimplementing them. +// The benchmark stack is still driven by shell. It orchestrates image builds +// and load-test deployment that are out of scope for this command, so +// ate-setup shells out to it with a translated argument list rather than +// reimplementing it. const deployLocustScript = "benchmarking/deploy_locust.sh" // BenchmarkOptions shapes the benchmark WorkerPool. @@ -66,15 +66,6 @@ func (e *Env) DeployBenchmarks(ctx context.Context, opts BenchmarkOptions) error return err } log.Stepf("deploy_benchmarks (worker_count=%d, sandbox_class=%s)", opts.WorkerCount, opts.SandboxClass) - - // The microvm SandboxConfig lives outside the default set installed by - // `deploy ate-system`, which only installs gvisor-default. The workloads - // deploy references it by name and would fail if this were skipped. - if opts.SandboxClass == config.SandboxClassMicrovm { - if err := e.DeployMicroVMDeps(ctx); err != nil { - return err - } - } return e.runScript(ctx, deployLocustScript, deployLocustArgs(opts, e.Cfg.OtlpEndpoint, e.Cfg.BenchmarkActorMemory)...) } @@ -106,18 +97,7 @@ func (e *Env) DeleteBenchmarks(ctx context.Context, opts BenchmarkOptions) error return err } log.Stepf("delete_benchmarks (sandbox_class=%s)", opts.SandboxClass) - - if err := e.runScript(ctx, deployLocustScript, "--delete"); err != nil { - return err - } - // The microvm SandboxConfig is cluster-wide. It is only torn down when - // the caller opted into microvm, and only after the benchmark pool is - // gone. The script leaves it in place if another pool still uses it. - // `delete all` tries again once every demo is gone. - if opts.SandboxClass == config.SandboxClassMicrovm { - return e.DeleteMicroVMDeps(ctx) - } - return nil + return e.runScript(ctx, deployLocustScript, "--delete") } // runScript executes a repository script from the repository root with the diff --git a/cmd/ate-setup/internal/steps/delete.go b/cmd/ate-setup/internal/steps/delete.go index 7223e8a332..98d6e1cf0f 100644 --- a/cmd/ate-setup/internal/steps/delete.go +++ b/cmd/ate-setup/internal/steps/delete.go @@ -65,6 +65,9 @@ func (e *Env) DeleteAteSystem(ctx context.Context) error { return err } + if err := e.DeleteMicroVMDeps(ctx); err != nil { + return err + } for _, path := range [][]string{ {"components", "agentgateway", "configmap.yaml"}, {"components", "fake-data-plane", "fake-workersync.yaml"}, @@ -207,38 +210,14 @@ type Deleter interface { Delete(ctx context.Context, e *Env) error } -// DeleteAll removes every demo, then the microvm SandboxConfig, then the -// control plane. +// DeleteAll removes every registered demo and then the control plane. func (e *Env) DeleteAll(ctx context.Context, demos []Deleter) error { log.Step("delete_all") - if err := e.deleteDemos(ctx, demos); err != nil { - return err - } - return e.DeleteAteSystem(ctx) -} - -// deleteDemos removes every demo, then tries to delete the microvm -// SandboxConfig. An earlier `delete benchmarks` or `delete demo` may have -// left the config behind because a pool still used it. After the demos, no -// pool does. -// -// The delete is best effort. The script skips a config that is still in use, -// and a failure is only logged, because DeleteAteSystem deletes the config -// with the CRD anyway. -// -// The gvisor config needs no such step. It is installed and deleted with the -// control plane, and DeleteAteSystem strips its finalizer itself. The microvm -// config is managed by hack/install-microvm-deps.sh, whose `kubectl delete` -// would block while a pool still uses it. -func (e *Env) deleteDemos(ctx context.Context, demos []Deleter) error { for _, demo := range demos { if err := demo.Delete(ctx, e); err != nil { return err } } - if err := e.runScript(ctx, installMicrovmDepScript, "--delete"); err != nil { - log.Warnf("while deleting the microvm SandboxConfig: %v", err) - } - return nil + return e.DeleteAteSystem(ctx) } diff --git a/cmd/ate-setup/internal/steps/delete_test.go b/cmd/ate-setup/internal/steps/delete_test.go index 9274cadc6f..a255a23b70 100644 --- a/cmd/ate-setup/internal/steps/delete_test.go +++ b/cmd/ate-setup/internal/steps/delete_test.go @@ -18,8 +18,6 @@ import ( "context" "errors" "fmt" - "os" - "path/filepath" "slices" "strings" "testing" @@ -37,7 +35,6 @@ import ( "k8s.io/client-go/util/retry" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" - "go.ate.dev/substrate/cmd/ate-setup/internal/config" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/internal/clustertrustbundle" ) @@ -193,92 +190,6 @@ func TestReleaseSandboxConfigs(t *testing.T) { } } -// recordingDeleter is a Deleter that appends its name to a log file shared -// with the fake microvm script, so a test can read back the order of calls. -type recordingDeleter struct { - name string - log string - err error -} - -func (d recordingDeleter) Delete(context.Context, *Env) error { - appendLine(d.log, d.name) - return d.err -} - -func appendLine(path, line string) { - f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil { - panic(err) - } - defer f.Close() - fmt.Fprintln(f, line) -} - -func TestDeleteDemos(t *testing.T) { - demoErr := errors.New("demo failed") - tests := []struct { - name string - // demos are the deleters in order. A nil entry's err is nil. - demos []recordingDeleter - // scriptExit is the exit code of the fake install-microvm-deps.sh. - scriptExit int - wantCalls []string - wantErr error - }{ - { - name: "microvm SandboxConfig is deleted after every demo", - demos: []recordingDeleter{{name: "demo-a"}, {name: "demo-b"}}, - wantCalls: []string{"demo-a", "demo-b", "script --delete"}, - }, - { - name: "script failure is logged, not returned", - demos: []recordingDeleter{{name: "demo-a"}}, - scriptExit: 1, - wantCalls: []string{"demo-a", "script --delete"}, - }, - { - name: "a failing demo stops before the script", - demos: []recordingDeleter{{name: "demo-a", err: demoErr}, {name: "demo-b"}}, - wantCalls: []string{"demo-a"}, - wantErr: demoErr, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - root := t.TempDir() - calls := filepath.Join(root, "calls.log") - script := filepath.Join(root, installMicrovmDepScript) - if err := os.MkdirAll(filepath.Dir(script), 0o755); err != nil { - t.Fatal(err) - } - body := fmt.Sprintf("#!/bin/sh\necho \"script $*\" >> %q\nexit %d\n", calls, tt.scriptExit) - if err := os.WriteFile(script, []byte(body), 0o755); err != nil { - t.Fatal(err) - } - demos := make([]Deleter, 0, len(tt.demos)) - for _, d := range tt.demos { - d.log = calls - demos = append(demos, d) - } - e := &Env{Cfg: &config.Config{Root: root}} - - err := e.deleteDemos(t.Context(), demos) - if !errors.Is(err, tt.wantErr) { - t.Fatalf("deleteDemos() error = %v, want %v", err, tt.wantErr) - } - got, _ := os.ReadFile(calls) - gotCalls := strings.Fields(strings.ReplaceAll(strings.TrimSpace(string(got)), "script --delete", "script_--delete")) - for i := range gotCalls { - gotCalls[i] = strings.ReplaceAll(gotCalls[i], "script_--delete", "script --delete") - } - if !slices.Equal(gotCalls, tt.wantCalls) { - t.Errorf("calls = %q, want %q", gotCalls, tt.wantCalls) - } - }) - } -} - // trustBundle returns a bundle for signer the way the API server requires it // to be named: the signer name with "/" replaced by ":", then a suffix. func trustBundle(signer, suffix string) *certsv1.ClusterTrustBundle { diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index e235e96b0a..5065378e8a 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -43,6 +43,9 @@ type DeployOptions struct { // SetupCSI additionally installs the CSI driver (nfs, hostpath, both, none). // Kind only. The hostpath driver is Kind only. SetupCSI string + // SkipMicroVMDeps skips staging the micro-VM guest assets and applying the + // microvm SandboxConfig. + SkipMicroVMDeps bool } // Validate checks the options that can be checked without configuration or a @@ -116,10 +119,6 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } - if err := e.DeploySandboxConfig(ctx); err != nil { - return err - } - // Ahead of the bundle below, for the same reason as the namespace: every // workload pulls this ConfigMap in via envFrom, and a container whose // envFrom target is missing will not start. @@ -152,6 +151,12 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } + // After the bundle on Kind, which creates the rustfs Deployment and + // bucket-init Job that DeployMicroVMDeps stages guest assets into. + if err := e.DeploySandboxConfig(ctx, opts.SkipMicroVMDeps); err != nil { + return err + } + // After the bundle, which resets the pod template to the sidecar-free base. if err := e.reconcileCloudSQLProxySidecar(ctx); err != nil { return err @@ -589,8 +594,8 @@ func (e *Env) DeployAtenet(ctx context.Context) error { } // DeploySandboxConfig applies the SandboxConfig admission policy, then the -// default gVisor SandboxConfig. -func (e *Env) DeploySandboxConfig(ctx context.Context) error { +// default gVisor and (unless skipMicroVMDeps is set) micro-VM SandboxConfigs. +func (e *Env) DeploySandboxConfig(ctx context.Context, skipMicroVMDeps bool) error { log.Step("deploy_sandboxconfig") if err := e.EnsureCRDs(ctx); err != nil { @@ -598,14 +603,20 @@ func (e *Env) DeploySandboxConfig(ctx context.Context) error { } // Enforce per-class SandboxConfig asset requirements. This is applied - // before any SandboxConfig so the config below is validated too. + // before any SandboxConfig so the configs below are validated too. if err := e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-validation.yaml")); err != nil { return err } - // Install the cluster-wide default gVisor SandboxConfig that gVisor - // WorkerPools resolve by default unless they pin their own configRef. - return e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-gvisor.yaml")) + // Install the cluster-wide default SandboxConfigs that WorkerPools + // resolve by default unless they pin their own configRef. + if err := e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-gvisor.yaml")); err != nil { + return err + } + if skipMicroVMDeps { + return nil + } + return e.DeployMicroVMDeps(ctx) } // EnsureCRDs creates the generated CRDs that are missing. Existing CRDs and diff --git a/cmd/ate-setup/internal/steps/microvm_test.go b/cmd/ate-setup/internal/steps/microvm_test.go index 11195c1d3e..5a0f9cfb2c 100644 --- a/cmd/ate-setup/internal/steps/microvm_test.go +++ b/cmd/ate-setup/internal/steps/microvm_test.go @@ -492,50 +492,6 @@ func TestWaitJobComplete(t *testing.T) { }) } -// Deploying benchmarks onto micro-VM has to bring the SandboxConfig with it: -// the workloads reference it by name. -func TestDeployBenchmarksInstallsMicroVMDeps(t *testing.T) { - silenceStepLog(t) - - // Passing an unsupported ARCH makes DeployMicroVMDeps return an error - // immediately, proving DeployBenchmarks invoked it before deploy_locust.sh. - t.Setenv("ARCH", "unsupported-arch") - env := &Env{Cfg: &config.Config{Root: t.TempDir(), Kind: true}} - - err := env.DeployBenchmarks(t.Context(), BenchmarkOptions{ - WorkerCount: 1, - SandboxClass: config.SandboxClassMicrovm, - }) - if err == nil || !strings.Contains(err.Error(), "unsupported ARCH=unsupported-arch") { - t.Fatalf("DeployBenchmarks() = %v, want error from DeployMicroVMDeps", err) - } -} - -// Confirm the opposite: a gvisor benchmark run must not touch the cluster-wide -// micro-VM SandboxConfig. -func TestDeleteBenchmarksLeavesMicroVMDepsAloneForGvisor(t *testing.T) { - silenceStepLog(t) - - root := t.TempDir() - locust := filepath.Join(root, deployLocustScript) - if err := os.MkdirAll(filepath.Dir(locust), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(locust, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { - t.Fatal(err) - } - - // Kube is nil: if DeleteBenchmarks called DeleteMicroVMDeps for gvisor, it - // would dereference e.Kube and panic. - env := &Env{Cfg: &config.Config{Root: root}} - if err := env.DeleteBenchmarks(t.Context(), BenchmarkOptions{ - WorkerCount: 1, - SandboxClass: config.SandboxClassGvisor, - }); err != nil { - t.Fatalf("DeleteBenchmarks(gvisor) = %v, want nil", err) - } -} - func TestMicroVMDepsStepsAreSeparableFromContext(t *testing.T) { silenceStepLog(t) diff --git a/demos/counter/README.md b/demos/counter/README.md index 40da79a08a..81f7d5a319 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -48,13 +48,18 @@ Run it and follow the printed next steps: KIND_CLUSTER_NAME= ./hack/run-microvm-demo-kind.sh ``` -To deploy onto an existing cluster instead, install the microVM deps and then -the demo: +To deploy onto an existing cluster where `ate-system` is already installed: ```bash -go run ./cmd/ate-setup deploy microvm-deps go run ./cmd/ate-setup deploy demo counter-microvm ``` + +Inspect the deployed template with: + +```bash +kubectl ate get actor-templates -a ate-demo-counter-microvm +kubectl ate get actor-template counter-microvm -a ate-demo-counter-microvm -o yaml +```
diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index d609850dbf..10c84f745d 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -18,8 +18,7 @@ # memory snapshot round-tripped. The ActorTemplate is not a CRD: it lives in # counter-microvm-template.yaml.tmpl and is created through the ate API with # `kubectl ate create actor-template`. The cluster-wide `microvm` -# SandboxConfig referenced by the pool is installed by -# hack/install-microvm-deps.sh +# SandboxConfig referenced by the pool is installed by `deploy ate-system`. apiVersion: v1 kind: Namespace diff --git a/demos/egress/egress-microvm.yaml.tmpl b/demos/egress/egress-microvm.yaml.tmpl index d4f589b596..edd1f199d7 100644 --- a/demos/egress/egress-microvm.yaml.tmpl +++ b/demos/egress/egress-microvm.yaml.tmpl @@ -17,8 +17,7 @@ # sandbox classes. The ActorTemplate is a substrate resource, not a CRD: it # lives in egress-microvm-template.yaml.tmpl and is created through the # ate API with `kubectl ate create actor-template`. The cluster-wide `microvm` -# SandboxConfig referenced by the pool is installed by -# hack/install-microvm-deps.sh +# SandboxConfig referenced by the pool is installed by `deploy ate-system`. apiVersion: v1 kind: Namespace diff --git a/docs/dev/microvm-local.md b/docs/dev/microvm-local.md index 329bf8b650..6032d47e90 100644 --- a/docs/dev/microvm-local.md +++ b/docs/dev/microvm-local.md @@ -56,12 +56,11 @@ kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.c ./hack/run-microvm-demo-kind.sh ``` -This is a one-shot bring-up: it deploys the control plane, installs the -cluster-wide microVM deps via `hack/install-microvm-deps.sh` — assembling the -guest runtime assets for your architecture (skipped if already present under -`bin/microvm-assets/`), staging them into the in-cluster rustfs bucket, and -applying the `microvm` `SandboxConfig` — then deploys the demo worker pool + -template. +This is a one-shot bring-up: it deploys the control plane (which assembles the +guest runtime assets for your architecture if not already present under +`bin/microvm-assets/`, stages them into the in-cluster rustfs bucket, and +applies the cluster-wide `microvm` `SandboxConfig`), then deploys the demo +worker pool + template. ### 4. Verify diff --git a/docs/dev/mitm-egress-test-runbook.md b/docs/dev/mitm-egress-test-runbook.md index 2d3a7c978c..21fdf0e4ee 100644 --- a/docs/dev/mitm-egress-test-runbook.md +++ b/docs/dev/mitm-egress-test-runbook.md @@ -25,13 +25,6 @@ kubectl config use-context kind-kind ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' ./hack/install-ate-kind.sh --deploy-ate-system ``` -For microVM, install its assets before deploying the fixture: - -```bash -NO_DEV_ENV=true ATE_INSTALL_KIND=true KUBECTL_CONTEXT=kind-kind \ - ./hack/install-microvm-deps.sh --install -``` - Deploy both egress ActorTemplates and add local-origin trust to the current gateway: diff --git a/docs/operator-install.md b/docs/operator-install.md index 86b46f6d5b..2a7a8ab190 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -319,6 +319,7 @@ demos, the [benchmarks](../benchmarking/README.md#settings), and the | `publish.overwrite` | `ATE_PUBLISH_OVERWRITE` | `--overwrite` | `false` | `publish release-images` | | `record.dir` | `ATE_RECORD_DIR` | `--record-dir` | — | (global) | | `rolloutTimeout` | `ATE_INSTALL_ROLLOUT_TIMEOUT` | `--rollout-timeout` | `1m0s` | (global) | +| `skipMicroVMDeps` | `ATE_INSTALL_SKIP_MICROVM_DEPS` | `--skip-microvm-deps` | `false` | `deploy ate-system`, `deploy sandboxconfig` | | `storage.bucketName` | `BUCKET_NAME` | `--storage-bucket-name` | — | (global) | | `storage.snapshotLocation` | `SNAPSHOT_LOCATION` | `--snapshot-location` | — | (global) | | `storage.transferGSAName` | `SNAPSHOT_TRANSFER_GSA_NAME` | `--snapshot-transfer-gsa-name` | — | (global) | @@ -354,7 +355,7 @@ does not create the bucket or grant access to it: run [tools/setup-gcp](../tools/setup-gcp/README.md)). setup-gcp also creates an asset bucket, `-assets`, where -`hack/install-microvm-deps.sh` stages microVM sandbox assets, and gives atelet +`ate-setup` stages microVM sandbox assets, and gives atelet read access to it. atelet gets no access to the snapshot bucket: every snapshot transfer runs on a token scoped to its actor, issued by ate-api-server's credential exchange. A GCP project set up before the asset bucket existed diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh deleted file mode 100755 index 2de37b27d4..0000000000 --- a/hack/install-microvm-deps.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Installs or deletes the cluster-wide micro-VM (kata + cloud-hypervisor) -# dependencies. -# -# This is a translation shim: cmd/ate-setup carries out the action (`deploy -# microvm-deps` / `delete microvm-deps`), and this script only maps the -# historical `--install` and `--delete` flags onto those commands so existing -# scripts and CI jobs keep working unchanged. -# -# The environment variables the installer reads -- BUCKET_NAME, KUBECTL_CONTEXT, -# PROJECT_ID, KO_DEFAULTPLATFORMS, ARCH, OUT, ATE_INSTALL_KIND, NO_DEV_ENV -- -# and .ate-dev-env.sh are read by ate-setup directly. - -set -o errexit -o nounset -o pipefail - -ROOT="$(git rev-parse --show-toplevel)" -cd "${ROOT}" - -usage() { - cat <&2; usage; exit 1 ;; - esac - shift -done - -if [[ -z "${action}" ]]; then - usage - exit 1 -fi - -go run ./cmd/ate-setup "${action}" microvm-deps diff --git a/hack/microvm-assets/README.md b/hack/microvm-assets/README.md deleted file mode 100644 index 06a3c74fc3..0000000000 --- a/hack/microvm-assets/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# MicroVM runtime assets + counter demo (kind, fetch-not-bake) - -The `microvm` runtime (`cmd/ateworker-microvm`, kata + cloud-hypervisor) fetches its -toolchain at runtime — nothing kata-specific is baked into the worker image. ateworker drives -the kata-agent directly (no kata shim, no containerd). Each actor container's rootfs is an -overlay of a read-only lower (the OCI image, served into the guest over virtio-fs by -`virtiofsd`) and a writable upper on a guest tmpfs, so `virtiofsd` is part of the asset -set. The asset set is four files: - -- `cloud-hypervisor` — the VMM binary (fetched from its release) -- `virtiofsd` — the virtio-fs daemon serving the RO lower (from kata-static) -- `vmlinux` — the guest kernel (from kata-static) -- `rootfs.img` — the guest rootfs image (from kata-static) - -`go run ./cmd/ate-setup deploy microvm-deps` (or the `hack/install-microvm-deps.sh --install` -shim) assembles the asset set for your node arch into `bin/microvm-assets/$ARCH`, stages it -into the cluster's object store (`rustfs` on kind, GCS on GKE), and applies the cluster-wide -`microvm` `SandboxConfig`. When `/dev/kvm` is available, `hack/create-kind-cluster.sh` mounts -it into the node; atelet then advertises it as a device, which is what places microVM -workers there. - -> [!TIP] -> `hack/run-microvm-demo.sh` (and `hack/run-microvm-demo-kind.sh` for kind) automates the -> full bring-up below (control plane, micro-VM assets + `SandboxConfig`, and demo apply) -> without editing committed files. - -## Steps (run on a KVM-capable Linux host matching the node arch) - -1. **Bring up the cluster + control plane:** - ```sh - hack/create-kind-cluster.sh # mounts /dev/kvm into the nodes - hack/install-ate-kind.sh --deploy-ate-system # control plane + rustfs (bucket: ate-snapshots) - ``` - -2. **Assemble and stage the micro-VM assets + `SandboxConfig`:** - ```sh - ATE_INSTALL_KIND=true hack/install-microvm-deps.sh --install - ``` - -3. **Apply the demo + drive it:** - ```sh - hack/install-ate-kind.sh --deploy-demo-counter-microvm - ``` - Create an actor from `counter-microvm`, hit the in-RAM counter to increment it, suspend - (checkpoint), resume on a different worker pod, and confirm the count continues — proving the - guest-memory snapshot round-tripped across pods. - -## Notes -- Assets must not be stored under a snapshot location (an ActorTemplate's - `snapshotConfig.location` followed by `/atespaces/`): the node's object-store plugin - refuses to serve anything there as an asset. The staging scripts use `kata-assets/`, - which is outside it. On GKE they stage into the asset bucket, `BUCKET_NAME-assets`, - the only bucket setup-gcp lets nodes read. On kind they share the rustfs bucket. -- `assets` is single-arch (unlike runsc's amd64/arm64): stage assets matching the node arch. diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 7a7bc83d2d..751018a337 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -19,15 +19,13 @@ # and calls this script), mirroring install-ate.sh / install-ate-kind.sh. # # Composes: -# 1. hack/install-ate.sh --deploy-ate-system (control plane) -# 2. go run ./cmd/ate-setup deploy microvm-deps (asset build/stage + cluster-wide -# microvm SandboxConfig) -# 3. Deploy the counter-microvm demo (worker pool manifest, atespace, and -# ActorTemplate through the ate API). +# 1. hack/install-ate.sh deploy ate-system (control plane + micro-VM assets and SandboxConfig) +# 2. hack/install-ate.sh deploy demo counter-microvm (worker pool manifest, atespace, and +# ActorTemplate through the ate API) # # Like the other hack scripts, this sources .ate-dev-env.sh for the cluster / # registry / bucket settings unless NO_DEV_ENV is set. The control plane deploy -# requires a credential provider selection, which reaches it only through the +# requires a credential provider selection, which reaches it through the # environment here: export ATE_CREDENTIAL_PROVIDER (e.g. '{"name":"k8s.io"}'; # see hack/install-ate.sh --help) before running this, the kind wrapper # included. @@ -35,12 +33,10 @@ # Env (most come from .ate-dev-env.sh): # KO_DOCKER_REPO (required) image registry, e.g. gcr.io/PROJECT/ate-images for # GKE or localhost:5001 for kind. -# BUCKET_NAME object store bucket for snapshots (default: ate-snapshots). On -# GKE the assets go to BUCKET_NAME-assets, see install-microvm-deps.sh. +# BUCKET_NAME object store bucket for snapshots (default: ate-snapshots on kind; +# required on GKE, where ate-setup stages assets to BUCKET_NAME-assets). # KUBECTL_CONTEXT (optional) kube context; threaded into install + ko apply + kubectl. # PROJECT_ID (optional) GCP project for the GCS asset upload (GKE path). -# ARCH target arch (default: from KO_DEFAULTPLATFORMS, else host arch). -# OUT asset dir (default: $PWD/bin/microvm-assets/$ARCH, gitignored). # ATE_INSTALL_KIND "true" for the kind path (stage assets to rustfs + install-ate-kind.sh); # default false uploads assets to GCS + uses install-ate.sh. @@ -58,7 +54,6 @@ fi # --- env / defaults --------------------------------------------------------- KO_DOCKER_REPO="${KO_DOCKER_REPO:-}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" -BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" if [[ $# -gt 0 ]]; then echo "Error: unknown argument $1" >&2 @@ -79,23 +74,17 @@ log() { echo -e "${COLOR_CYAN}[run-microvm-demo]: $*${COLOR_RESET}" } -# --- 1. deploy the control plane ------------------------------------------- +# --- 1. deploy the control plane (includes micro-VM assets + SandboxConfig) - log "Deploying the ate control plane (deploy ate-system)..." if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. + # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ATE_INSTALL_KIND itself. KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh deploy ate-system else # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh deploy ate-system fi -# --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- -# `ate-setup deploy microvm-deps` handles the assemble/stage/apply flow. Ordering -# matters: the control plane must be up so the SandboxConfig CRD exists. -log "Installing micro-VM dependencies..." -KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" go run ./cmd/ate-setup deploy microvm-deps - -# --- 3. apply the demo ------------------------------------------------------ +# --- 2. apply the demo ------------------------------------------------------ KCTX_FLAG="" if [[ -n "${KUBECTL_CONTEXT}" ]]; then KCTX_FLAG=" --context=${KUBECTL_CONTEXT}" @@ -104,7 +93,7 @@ fi # The demo handler applies the worker pool, creates the atespace and the # ActorTemplate through the ate API, and waits for the golden snapshot; # dispatch through install-ate.sh like step 1. -log "Deploying the counter-microvm demo (--deploy-demo-counter-microvm)..." +log "Deploying the counter-microvm demo (deploy demo counter-microvm)..." if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh deploy demo counter-microvm else diff --git a/internal/e2e/suites/egressmitm/egressmitm_test.go b/internal/e2e/suites/egressmitm/egressmitm_test.go index 2d6da343b3..de3e43889c 100644 --- a/internal/e2e/suites/egressmitm/egressmitm_test.go +++ b/internal/e2e/suites/egressmitm/egressmitm_test.go @@ -63,9 +63,6 @@ var probeNamespace string // hack/setup-e2e-egress-tls-kind.sh // hack/run-e2e-kind.sh ./internal/e2e/suites/egressmitm -v -args --no-color // E2E_SANDBOX_CLASS=microvm hack/run-e2e-kind.sh ./internal/e2e/suites/egressmitm -v -args --no-color -// -// The micro-VM variant additionally needs the micro-VM deps installed -// (hack/run-microvm-demo-kind.sh, or hack/install-microvm-deps.sh --install). func TestActorEgressMITMTrust(t *testing.T) { env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") if err != nil { diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index 3a0f252440..1f43b51354 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -13,13 +13,11 @@ # limitations under the License. # Cluster-wide SandboxConfig for the micro-VM (kata + cloud-hypervisor) sandbox -# class. Unlike sandboxconfig-gvisor.yaml (applied unconditionally by -# hack/install-ate.sh --deploy-ate-system), this is opt-in: apply via -# hack/install-microvm-deps.sh --install (or `go run ./cmd/ate-setup deploy -# microvm-deps`), which assembles the asset set, stages it to the cluster's -# object store, and applies this manifest. A microvm WorkerPool resolves it as -# its class default (or names it via configRef: microvm), so a missing/stale -# config shows on the pool's status. +# class. `go run ./cmd/ate-setup deploy ate-system` (and `deploy sandboxconfig`) +# assembles the asset set, stages it to the cluster's object store, and applies +# this manifest alongside sandboxconfig-gvisor.yaml. A microvm WorkerPool +# resolves it as its class default (or names it via configRef: microvm), so a +# missing/stale config shows on the pool's status. # # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs) # are FETCHED at runtime from the asset bucket ${ASSET_BUCKET_NAME} under @@ -34,7 +32,7 @@ # The per-arch sha256 values below are the asset sets assembled by # cmd/ate-setup/internal/steps/microvm.go; atelet selects the block matching # the node's architecture, and each cluster's bucket holds that arch's binaries -# at these paths (staged by hack/install-microvm-deps.sh --install). +# at these paths. apiVersion: ate.dev/v1 kind: SandboxConfig @@ -63,7 +61,6 @@ spec: kata-image: url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/rootfs.img" sha256: "69f9a76e5a2563b7c74d1f271f3a904240f50ef405465f832021525894519f97" - # amd64 assets are kata 4.3.1 (ARCH=amd64). amd64: cloud-hypervisor: url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/cloud-hypervisor" diff --git a/tools/setup-gcp/README.md b/tools/setup-gcp/README.md index f0c685de76..db70000f57 100644 --- a/tools/setup-gcp/README.md +++ b/tools/setup-gcp/README.md @@ -329,7 +329,7 @@ bucket. Upgrade Substrate first, to a release whose node plugin gets snapshot credentials from ate-api-server's credential exchange. On an older release, removing these grants makes every suspend and resume fail with `403`. Remove those grants after running `create bucket` and `create iam`, and after -rerunning `hack/install-microvm-deps.sh --install` if you use microVM +rerunning `go run ./cmd/ate-setup deploy sandboxconfig` if you use microVM workers, so the assets are in the asset bucket. `ATELET` is set as in the manual equivalent above: diff --git a/tools/setup-gcp/cmd/common.go b/tools/setup-gcp/cmd/common.go index 9eb30093f0..4fb7fc45b0 100644 --- a/tools/setup-gcp/cmd/common.go +++ b/tools/setup-gcp/cmd/common.go @@ -123,7 +123,7 @@ func getEnv[T getEnvType](key string, fallback T) T { } // assetBucketName is the bucket for micro-VM sandbox assets. -// hack/install-microvm-deps.sh derives the same name. +// cmd/ate-setup derives the same name. func (c *Config) assetBucketName() string { return c.BucketName + "-assets" } From a8df78ea4bd3cbb88ca402d873ca0cd20175f200 Mon Sep 17 00:00:00 2001 From: Noah West Date: Wed, 14 Oct 2026 15:57:48 -0700 Subject: [PATCH 04/10] Stage multi-arch micro-VM assets and verify SHA-256 digests Stage both amd64 and arm64 guest runtime assets under kata-assets// in the object store and update manifests/microvm/sandboxconfig-microvm.yaml.tmpl to reference the per-arch URLs so mixed-architecture clusters and cross-architecture workstations do not collide on a single prefix. Verify the SHA-256 digest of each assembled or cached asset against the digests pinned in sandboxconfig-microvm.yaml.tmpl before staging, drop un-registered raw environment overrides from steps/microvm.go, and fold the remaining notes from hack/microvm-assets/README.md into docs/dev/microvm-local.md. Signed-off-by: Noah West --- cmd/ate-setup/internal/cmd/deploy_test.go | 4 +- cmd/ate-setup/internal/steps/microvm.go | 207 ++++++++++-------- cmd/ate-setup/internal/steps/microvm_test.go | 98 +++++++-- docs/api-guide.md | 2 +- docs/dev/microvm-local.md | 32 ++- .../microvm/sandboxconfig-microvm.yaml.tmpl | 30 +-- 6 files changed, 235 insertions(+), 138 deletions(-) diff --git a/cmd/ate-setup/internal/cmd/deploy_test.go b/cmd/ate-setup/internal/cmd/deploy_test.go index eb1e1dc72f..3fd579f36f 100644 --- a/cmd/ate-setup/internal/cmd/deploy_test.go +++ b/cmd/ate-setup/internal/cmd/deploy_test.go @@ -17,8 +17,8 @@ package cmd import ( "testing" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) // An upgrade or a rollback moves the control plane one component at a time, diff --git a/cmd/ate-setup/internal/steps/microvm.go b/cmd/ate-setup/internal/steps/microvm.go index 147f29d15f..def655a719 100644 --- a/cmd/ate-setup/internal/steps/microvm.go +++ b/cmd/ate-setup/internal/steps/microvm.go @@ -17,7 +17,6 @@ package steps import ( "archive/tar" "bufio" - "cmp" "context" "crypto/sha256" "fmt" @@ -28,7 +27,6 @@ import ( "os/exec" "path" "path/filepath" - "runtime" "strings" "time" @@ -41,9 +39,9 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/render" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/render" ) const ( @@ -54,7 +52,7 @@ const ( defaultKataVersion = "4.1.0" defaultCloudHypervisorVersion = "v53.0" // virtiofsdVersion is the version bundled inside kata-static for - // defaultKataVersion (not env-overridable; verified before stamping). + // defaultKataVersion (verified before stamping). virtiofsdVersion = "1.14.0" assetStampFile = ".asset-versions" @@ -73,7 +71,27 @@ const ( maxMicroVMAssetBytes int64 = 8 << 30 ) -var microvmAssets = []string{"cloud-hypervisor", "virtiofsd", "vmlinux", "rootfs.img"} +var ( + microvmArches = []string{"amd64", "arm64"} + microvmAssets = []string{"cloud-hypervisor", "virtiofsd", "vmlinux", "rootfs.img"} + + // microvmAssetDigests pins the expected sha256 digest of each assembled + // file in microvmAssets, matching manifests/microvm/sandboxconfig-microvm.yaml.tmpl. + microvmAssetDigests = map[string]map[string]string{ + "arm64": { + "cloud-hypervisor": "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850", + "virtiofsd": "3aa9240045d2261930ef04138fe05cf288f94c867e63e5d0845758bac5e3d458", + "vmlinux": "4960f6cf781274eaa445fc9f88bb8c5ce951649278bc9caa13afb1e99cfe80e6", + "rootfs.img": "7e3bdd500eb92de17c377bd84f8ed57e50d6bb68c7ec08c88f5e1916cd49f30d", + }, + "amd64": { + "cloud-hypervisor": "448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc", + "virtiofsd": "15b2e72a78cc08a9bd8a6943e89fb69c88cb3cbeb63069efceade835342ac7d4", + "vmlinux": "8e9dbc3a6e4c26adb089d23d31201edebce905c7f3e31aa67b32bdd41df1b86f", + "rootfs.img": "96497f64da1de9c7473fef46c3d29ddd0805d334731cf9d903a21a5b2c33cefb", + }, + } +) var sandboxConfigGVK = schema.GroupVersionKind{ Group: "ate.dev", @@ -84,48 +102,47 @@ var sandboxConfigGVK = schema.GroupVersionKind{ // microvmReleaseBaseURL is overridden in unit tests to point at an httptest.Server. var microvmReleaseBaseURL = "https://github.com" -// DeployMicroVMDeps installs the micro-VM prerequisites: the guest assets and -// the cluster-wide `microvm` SandboxConfig. +// DeployMicroVMDeps installs the micro-VM prerequisites: the guest assets for +// each supported architecture and the cluster-wide `microvm` SandboxConfig. func (e *Env) DeployMicroVMDeps(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } log.Step("deploy_microvm_deps") - arch := e.microvmArch() - if _, err := cloudHypervisorAssetName(arch); err != nil { - return err - } bucket, err := e.microVMBucket() if err != nil { return err } - outDir := cmp.Or(e.getenv("OUT"), e.Cfg.Path("bin", "microvm-assets", arch)) - kataVer := cmp.Or(e.getenv("KATA_VER"), defaultKataVersion) - chVer := cmp.Or(e.getenv("CH_VER"), defaultCloudHypervisorVersion) - wantStamp := microvmAssetStamp(arch, kataVer, chVer) - needAssemble, stale := microvmAssetsNeedAssemble(outDir, wantStamp) - if stale { - log.Infof("Asset set in %s is stale (version stamp mismatch); re-assembling.", outDir) - } - if needAssemble { - log.Infof("Assembling micro-VM assets into %s (ARCH=%s)...", outDir, arch) - if err := assembleMicroVMAssets(ctx, outDir, arch, kataVer, chVer); err != nil { + for _, arch := range microvmArches { + outDir := e.Cfg.Path("bin", "microvm-assets", arch) + wantStamp := microvmAssetStamp(arch, defaultKataVersion, defaultCloudHypervisorVersion) + needAssemble, stale := microvmAssetsNeedAssemble(outDir, wantStamp) + if stale { + log.Infof("Asset set in %s is stale (version stamp mismatch); re-assembling.", outDir) + } + if needAssemble { + log.Infof("Assembling micro-VM assets into %s (arch=%s)...", outDir, arch) + if err := assembleMicroVMAssets(ctx, outDir, arch, defaultKataVersion, defaultCloudHypervisorVersion); err != nil { + return err + } + } else { + log.Infof("Assets already present in %s; skipping assemble.", outDir) + } + if err := verifyMicroVMAssetDigests(outDir, arch); err != nil { return err } - } else { - log.Infof("Assets already present in %s; skipping assemble.", outDir) } if e.Cfg.Kind { log.Infof("Staging assets to in-cluster rustfs bucket %s (kata-assets/)...", bucket) - if err := e.stageMicroVMAssetsToRustfs(ctx, outDir, bucket); err != nil { + if err := e.stageMicroVMAssetsToRustfs(ctx, bucket); err != nil { return err } } else { log.Infof("Uploading assets to gs://%s/kata-assets/ ...", bucket) - if err := e.stageMicroVMAssetsToGCS(ctx, outDir, bucket); err != nil { + if err := e.stageMicroVMAssetsToGCS(ctx, bucket); err != nil { return err } } @@ -197,35 +214,6 @@ func (e *Env) microVMBucketScheme() string { return "gs" } -func (e *Env) microvmArch() string { - if arch := e.getenv("ARCH"); arch != "" { - return arch - } - if e.Cfg != nil && e.Cfg.KODefaultPlatforms != "" { - if _, arch, ok := strings.Cut(e.Cfg.KODefaultPlatforms, "/"); ok { - return arch - } - return e.Cfg.KODefaultPlatforms - } - return runtime.GOARCH -} - -func (e *Env) getenv(key string) string { - if v := os.Getenv(key); v != "" { - return v - } - if e.Cfg == nil { - return "" - } - prefix := key + "=" - for _, kv := range e.Cfg.ScriptEnv() { - if val, ok := strings.CutPrefix(kv, prefix); ok { - return val - } - } - return "" -} - func cloudHypervisorAssetName(arch string) (string, error) { switch arch { case "arm64": @@ -233,7 +221,7 @@ func cloudHypervisorAssetName(arch string) (string, error) { case "amd64": return "cloud-hypervisor-static", nil default: - return "", fmt.Errorf("unsupported ARCH=%s", arch) + return "", fmt.Errorf("unsupported arch %q", arch) } } @@ -251,6 +239,37 @@ func checkMicroVMAssets(outDir string) error { return nil } +func verifyMicroVMAssetDigests(outDir, arch string) error { + digests, ok := microvmAssetDigests[arch] + if !ok { + return fmt.Errorf("no pinned micro-VM asset digests for arch %q", arch) + } + for _, f := range microvmAssets { + p := filepath.Join(outDir, f) + got, err := fileSHA256(p) + if err != nil { + return err + } + if want := digests[f]; got != want { + return fmt.Errorf("micro-VM asset %s sha256 = %s, want %s (pinned in %s)", p, got, want, microvmSandboxConfigTemplate) + } + } + return nil +} + +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + func microvmAssetsNeedAssemble(outDir, wantStamp string) (needAssemble, stale bool) { if err := checkMicroVMAssets(outDir); err != nil { return true, false @@ -310,19 +329,13 @@ func assembleMicroVMAssets(ctx context.Context, outDir, arch, kataVer, chVer str return fmt.Errorf("while writing %s: %w", stampPath, err) } - log.Infof(">> sha256 (paste all four into the per-arch block in %s):", microvmSandboxConfigTemplate) + log.Infof(">> sha256 (paste all four into the %s block in %s):", arch, microvmSandboxConfigTemplate) for _, f := range microvmAssets { - file, err := os.Open(filepath.Join(outDir, f)) + sum, err := fileSHA256(filepath.Join(outDir, f)) if err != nil { return err } - h := sha256.New() - _, copyErr := io.Copy(h, file) - _ = file.Close() - if copyErr != nil { - return copyErr - } - log.Infof("%x %s", h.Sum(nil), f) + log.Infof("%s %s", sum, f) } return nil } @@ -352,6 +365,13 @@ func downloadAndExtractKataAssets(ctx context.Context, kataURL, workDir, outDir mode fs.FileMode }{ {archivePath: kataKernelSymlink, outName: "vmlinux", mode: 0o644}, + // rootfs.img is attached over virtio-pmem and mounted with DAX (cmd/ateworker-microvm/run.go). + // The image must meet the following requirements: + // - a size that is a multiple of 2 MiB (cloud-hypervisor) + // - partition 1 aligned to the guest page size + // - a filesystem block size equal to the guest page size + // The kata-containers.img used here is built for this, and vmlinux.container uses 4 KiB pages. + // See https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder/image-builder/image_builder.sh {archivePath: kataRootfsSymlink, outName: "rootfs.img", mode: 0o644}, {archivePath: kataLibexecVirtiofsd, outName: "virtiofsd", mode: 0o755}, } { @@ -489,7 +509,7 @@ func verifyVirtiofsdVersion(ctx context.Context, virtiofsdPath, kataVer, wantVer return nil } -func (e *Env) stageMicroVMAssetsToRustfs(ctx context.Context, outDir, bucket string) error { +func (e *Env) stageMicroVMAssetsToRustfs(ctx context.Context, bucket string) error { if _, err := exec.LookPath("docker"); err != nil { return fmt.Errorf("'docker' is required to run the S3 client but was not found in PATH: %w", err) } @@ -517,13 +537,14 @@ func (e *Env) stageMicroVMAssetsToRustfs(ctx context.Context, outDir, bucket str } endpoint := fmt.Sprintf("http://%s:9000", svc.Spec.ClusterIP) + // Credentials and region are fixed in manifests/ate-install/kind/rustfs.yaml. awsCLI := func(stdin io.Reader, args ...string) error { dockerArgs := append([]string{ "run", "--rm", "-i", "--network", "container:" + node, - "-e", "AWS_ACCESS_KEY_ID=" + cmp.Or(e.getenv("AWS_ACCESS_KEY_ID"), "rustfsadmin"), - "-e", "AWS_SECRET_ACCESS_KEY=" + cmp.Or(e.getenv("AWS_SECRET_ACCESS_KEY"), "rustfsadmin"), - "-e", "AWS_REGION=" + cmp.Or(e.getenv("AWS_REGION"), "us-east-1"), + "-e", "AWS_ACCESS_KEY_ID=rustfsadmin", + "-e", "AWS_SECRET_ACCESS_KEY=rustfsadmin", + "-e", "AWS_REGION=us-east-1", "-e", "AWS_ENDPOINT_URL=" + endpoint, rustfsAWSCLIImage, }, args...) @@ -531,21 +552,24 @@ func (e *Env) stageMicroVMAssetsToRustfs(ctx context.Context, outDir, bucket str } log.Infof(">> Uploading assets to s3://%s/kata-assets/ via %s (netns of %s)...", bucket, endpoint, node) - for _, f := range microvmAssets { - log.Infof(" %s", f) - file, err := os.Open(filepath.Join(outDir, f)) - if err != nil { - return err - } - err = awsCLI(file, "s3", "cp", "-", fmt.Sprintf("s3://%s/kata-assets/%s", bucket, f)) - _ = file.Close() - if err != nil { - return fmt.Errorf("while uploading %s to rustfs: %w", f, err) + for _, arch := range microvmArches { + outDir := e.Cfg.Path("bin", "microvm-assets", arch) + for _, f := range microvmAssets { + log.Infof(" %s/%s", arch, f) + file, err := os.Open(filepath.Join(outDir, f)) + if err != nil { + return err + } + err = awsCLI(file, "s3", "cp", "-", fmt.Sprintf("s3://%s/kata-assets/%s/%s", bucket, arch, f)) + _ = file.Close() + if err != nil { + return fmt.Errorf("while uploading %s/%s to rustfs: %w", arch, f, err) + } } } log.Infof(">> Done. Verify:") - return awsCLI(nil, "s3", "ls", fmt.Sprintf("s3://%s/kata-assets/", bucket)) + return awsCLI(nil, "s3", "ls", "--recursive", fmt.Sprintf("s3://%s/kata-assets/", bucket)) } func (e *Env) waitJobComplete(ctx context.Context, namespace, name string, timeout time.Duration) error { @@ -570,23 +594,26 @@ func (e *Env) waitJobComplete(ctx context.Context, namespace, name string, timeo }) } -func (e *Env) stageMicroVMAssetsToGCS(ctx context.Context, outDir, bucket string) error { +func (e *Env) stageMicroVMAssetsToGCS(ctx context.Context, bucket string) error { var projectFlag []string if e.Cfg.ProjectID != "" { projectFlag = []string{"--project=" + e.Cfg.ProjectID} } - for _, f := range microvmAssets { - log.Infof(" %s", f) - args := append(append([]string{"storage", "cp"}, projectFlag...), - filepath.Join(outDir, f), fmt.Sprintf("gs://%s/kata-assets/%s", bucket, f)) - if err := e.runCmd(ctx, nil, "gcloud", args...); err != nil { - return fmt.Errorf("while uploading %s to GCS: %w", f, err) + for _, arch := range microvmArches { + outDir := e.Cfg.Path("bin", "microvm-assets", arch) + for _, f := range microvmAssets { + log.Infof(" %s/%s", arch, f) + args := append(append([]string{"storage", "cp"}, projectFlag...), + filepath.Join(outDir, f), fmt.Sprintf("gs://%s/kata-assets/%s/%s", bucket, arch, f)) + if err := e.runCmd(ctx, nil, "gcloud", args...); err != nil { + return fmt.Errorf("while uploading %s/%s to GCS: %w", arch, f, err) + } } } log.Infof(">> Done. Verify:") - lsArgs := append(append([]string{"storage", "ls"}, projectFlag...), fmt.Sprintf("gs://%s/kata-assets/", bucket)) + lsArgs := append(append([]string{"storage", "ls", "--recursive"}, projectFlag...), fmt.Sprintf("gs://%s/kata-assets/", bucket)) return e.runCmd(ctx, nil, "gcloud", lsArgs...) } diff --git a/cmd/ate-setup/internal/steps/microvm_test.go b/cmd/ate-setup/internal/steps/microvm_test.go index 5a0f9cfb2c..d0092d54cf 100644 --- a/cmd/ate-setup/internal/steps/microvm_test.go +++ b/cmd/ate-setup/internal/steps/microvm_test.go @@ -18,13 +18,14 @@ import ( "archive/tar" "bytes" "context" + "crypto/sha256" + "encoding/hex" "errors" "io" "net/http" "net/http/httptest" "os" "path/filepath" - "runtime" "strings" "testing" "time" @@ -34,10 +35,10 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/render" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/render" ) func silenceStepLog(t *testing.T) { @@ -46,28 +47,69 @@ func silenceStepLog(t *testing.T) { t.Cleanup(func() { log.SetOutput(os.Stdout) }) } -func TestMicroVMConfigResolution(t *testing.T) { - t.Run("ARCH env beats KO_DEFAULTPLATFORMS and GOARCH", func(t *testing.T) { - t.Setenv("ARCH", "arm64") - e := &Env{Cfg: &config.Config{KODefaultPlatforms: "linux/amd64"}} - if got := e.microvmArch(); got != "arm64" { - t.Errorf("microvmArch() = %q, want arm64", got) +func sha256HexString(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +func TestVerifyMicroVMAssetDigests(t *testing.T) { + origDigests := microvmAssetDigests["amd64"] + microvmAssetDigests["amd64"] = map[string]string{ + "cloud-hypervisor": sha256HexString("ch-bytes"), + "virtiofsd": sha256HexString("virtiofsd-bytes"), + "vmlinux": sha256HexString("vmlinux-bytes"), + "rootfs.img": sha256HexString("rootfs-bytes"), + } + t.Cleanup(func() { microvmAssetDigests["amd64"] = origDigests }) + + writeValidAssets := func(t *testing.T, dir string) { + t.Helper() + for name, body := range map[string]string{ + "cloud-hypervisor": "ch-bytes", + "virtiofsd": "virtiofsd-bytes", + "vmlinux": "vmlinux-bytes", + "rootfs.img": "rootfs-bytes", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + } + + t.Run("matching digests succeed", func(t *testing.T) { + dir := t.TempDir() + writeValidAssets(t, dir) + if err := verifyMicroVMAssetDigests(dir, "amd64"); err != nil { + t.Fatalf("verifyMicroVMAssetDigests() = %v, want nil", err) + } + }) + + t.Run("mismatched digest fails", func(t *testing.T) { + dir := t.TempDir() + writeValidAssets(t, dir) + if err := os.WriteFile(filepath.Join(dir, "vmlinux"), []byte("corrupted"), 0o644); err != nil { + t.Fatal(err) + } + err := verifyMicroVMAssetDigests(dir, "amd64") + if err == nil || !strings.Contains(err.Error(), "sha256 =") { + t.Fatalf("verifyMicroVMAssetDigests() = %v, want sha256 mismatch error", err) } }) - t.Run("KO_DEFAULTPLATFORMS suffix beats GOARCH", func(t *testing.T) { - t.Setenv("ARCH", "") - e := &Env{Cfg: &config.Config{KODefaultPlatforms: "linux/arm64"}} - if got := e.microvmArch(); got != "arm64" { - t.Errorf("microvmArch() = %q, want arm64", got) + t.Run("missing file fails", func(t *testing.T) { + dir := t.TempDir() + writeValidAssets(t, dir) + if err := os.Remove(filepath.Join(dir, "rootfs.img")); err != nil { + t.Fatal(err) + } + if err := verifyMicroVMAssetDigests(dir, "amd64"); err == nil { + t.Fatal("verifyMicroVMAssetDigests() succeeded with missing file, want error") } }) - t.Run("falls back to runtime.GOARCH", func(t *testing.T) { - t.Setenv("ARCH", "") - e := &Env{Cfg: &config.Config{}} - if got := e.microvmArch(); got != runtime.GOARCH { - t.Errorf("microvmArch() = %q, want %q", got, runtime.GOARCH) + t.Run("unknown arch fails", func(t *testing.T) { + if err := verifyMicroVMAssetDigests(t.TempDir(), "riscv64"); err == nil { + t.Fatal("verifyMicroVMAssetDigests() succeeded with unknown arch, want error") } }) } @@ -383,9 +425,17 @@ func TestMicroVMSandboxConfigTemplate(t *testing.T) { if strings.Contains(string(rendered), "${") { t.Errorf("rendered manifest still contains unexpanded placeholder:\n%s", rendered) } - wantURL := tc.scheme + "://" + tc.bucket + "/kata-assets/cloud-hypervisor" - if !strings.Contains(string(rendered), wantURL) { - t.Errorf("rendered manifest missing expected bucket URL %q:\n%s", wantURL, rendered) + for _, arch := range microvmArches { + for _, asset := range microvmAssets { + wantURL := tc.scheme + "://" + tc.bucket + "/kata-assets/" + arch + "/" + asset + if !strings.Contains(string(rendered), wantURL) { + t.Errorf("rendered manifest missing expected bucket URL %q:\n%s", wantURL, rendered) + } + wantDigest := microvmAssetDigests[arch][asset] + if !strings.Contains(string(rendered), wantDigest) { + t.Errorf("rendered manifest missing expected %s/%s digest %q:\n%s", arch, asset, wantDigest, rendered) + } + } } objs, err := kube.DecodeManifestBytes(rendered) if err != nil { diff --git a/docs/api-guide.md b/docs/api-guide.md index ffb19a45ff..0773468226 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -564,7 +564,7 @@ spec: A `microvm` `SandboxConfig` supplies the [Kata Containers](https://katacontainers.io/) + [Cloud Hypervisor](https://www.cloudhypervisor.org/) toolchain instead of `runsc`. Each architecture of each version must define the full asset set — `cloud-hypervisor`, `virtiofsd`, `kata-kernel`, and `kata-image` — which a `ValidatingAdmissionPolicy` enforces at apply time. Worker pods for a microVM pool require `/dev/kvm` and nested-virtualization-capable nodes. The controller requests those devices on the pod automatically, and atelet advertises them only where they exist, so placement follows the hardware rather than a node label. Clusters that reserve nested-virt nodes with an `ate.dev/sandboxClass=microvm` taint are still tolerated: advertising a device attracts these pods to capable nodes but repels nothing else from them. The same convention applies to every class: worker pods of a pool tolerate `ate.dev/sandboxClass=:NoSchedule`, so a cluster can reserve a node pool per sandbox class with that taint, and the atelet DaemonSet tolerates the key for any value. On GKE, a pool can also select a ComputeClass with `template.nodeSelector` (`cloud.google.com/compute-class: `): GKE taints the nodes it creates for the class `cloud.google.com/compute-class=:NoSchedule` and adds the matching toleration to the worker pods, and the atelet DaemonSet tolerates that key as well. -See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. +`go run ./cmd/ate-setup deploy ate-system` (and `deploy sandboxconfig`) assembles and stages these assets automatically; see [`docs/dev/microvm-local.md`](dev/microvm-local.md) for local bring-up and a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. --- diff --git a/docs/dev/microvm-local.md b/docs/dev/microvm-local.md index 6032d47e90..8c09b13eca 100644 --- a/docs/dev/microvm-local.md +++ b/docs/dev/microvm-local.md @@ -13,8 +13,28 @@ Complete the [Quickstart (Development)](../../README.md#quickstart-development) in the README first — it covers the base tooling and the default (gVisor) path this guide builds on. For background on the runtime, see -[architecture.md](../architecture.md) and -[hack/microvm-assets/README.md](../../hack/microvm-assets/README.md). +[architecture.md](../architecture.md). + +The `microvm` runtime (`cmd/ateworker-microvm`) fetches its guest toolchain at +runtime rather than baking Kata into the worker image. Each actor container's +rootfs is an overlay of a read-only lower (the OCI image, served into the guest +over virtio-fs by `virtiofsd`) and a writable upper on a guest tmpfs, so the +asset set is four files per architecture (`amd64` and `arm64`): + +- `cloud-hypervisor` — the VMM binary (from its release) +- `virtiofsd` — the virtio-fs daemon serving the read-only lower (from `kata-static`) +- `vmlinux` — the guest kernel (from `kata-static`) +- `rootfs.img` — the guest rootfs image (from `kata-static`) + +`go run ./cmd/ate-setup deploy ate-system` (and `deploy sandboxconfig`) +assembles both architectures into `bin/microvm-assets/{amd64,arm64}/`, verifies +their SHA-256 digests against `manifests/microvm/sandboxconfig-microvm.yaml.tmpl`, +stages them into the cluster's object store under `kata-assets//` (the +shared `rustfs` bucket on kind, and `-assets` on GKE, the only +bucket `setup-gcp` lets nodes read), and applies the cluster-wide `microvm` +`SandboxConfig`. Assets must not be stored under a snapshot location (an +ActorTemplate's `snapshotConfig.location` followed by `/atespaces/`), as the +node's object-store plugin refuses to serve anything there as an asset. ## Option A: Linux host with KVM @@ -57,7 +77,7 @@ kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.c ``` This is a one-shot bring-up: it deploys the control plane (which assembles the -guest runtime assets for your architecture if not already present under +guest runtime assets for `amd64` and `arm64` if not already present under `bin/microvm-assets/`, stages them into the in-cluster rustfs bucket, and applies the cluster-wide `microvm` `SandboxConfig`), then deploys the demo worker pool + template. @@ -158,9 +178,9 @@ cd ./hack/run-microvm-demo-kind.sh ``` -The first run of `run-microvm-demo-kind.sh` downloads about 590 MB of guest -runtime assets and extracts ~285 MB into `bin/microvm-assets/arm64/`; later runs -reuse that directory when the version stamp matches. +The first run of `run-microvm-demo-kind.sh` downloads the guest runtime +releases and extracts them into `bin/microvm-assets/{amd64,arm64}/`; later runs +reuse those directories when the version stamps and SHA-256 digests match. Verify as in Option A, step 4. diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index 1f43b51354..c928992790 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -21,18 +21,18 @@ # # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs) # are FETCHED at runtime from the asset bucket ${ASSET_BUCKET_NAME} under -# kata-assets/ (the rustfs bucket on kind, its own GCS bucket on GKE), through -# the node's object-store plugin - NOT baked into the worker image. +# kata-assets// (the rustfs bucket on kind, its own GCS bucket on GKE), +# through the node's object-store plugin - NOT baked into the worker image. # ${BUCKET_SCHEME} is that store's URL scheme (s3 on kind, gs on GKE): the # plugin serves only its own backend's scheme. The assets must stay outside any # snapshot location, which the plugin refuses to serve as assets. ateworker -# boots cloud-hypervisor itself and gives the actor an overlay rootfs (virtio-fs -# RO lower + guest-tmpfs upper), so it fetches virtiofsd too. +# boots cloud-hypervisor itself and gives the actor an overlay rootfs +# (virtio-fs RO lower + guest-tmpfs upper), so it fetches virtiofsd too. # -# The per-arch sha256 values below are the asset sets assembled by +# The per-arch sha256 values below are the asset sets assembled and verified by # cmd/ate-setup/internal/steps/microvm.go; atelet selects the block matching -# the node's architecture, and each cluster's bucket holds that arch's binaries -# at these paths. +# the node's architecture, and ate-setup stages both architectures' binaries at +# these paths. apiVersion: ate.dev/v1 kind: SandboxConfig @@ -49,28 +49,28 @@ spec: assets: arm64: cloud-hypervisor: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/cloud-hypervisor" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/arm64/cloud-hypervisor" sha256: "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850" # virtiofsd serves the overlay RO lower (virtio-fs). virtiofsd: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/virtiofsd" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/arm64/virtiofsd" sha256: "4b43f4f6de6500203b01214a790314eea4182f5f110594ddb28ae2805622f06f" kata-kernel: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/vmlinux" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/arm64/vmlinux" sha256: "dece388830720fe8bddb41552fff5f486240aeece5acc0aebc1812761e3e3f66" kata-image: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/rootfs.img" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/arm64/rootfs.img" sha256: "69f9a76e5a2563b7c74d1f271f3a904240f50ef405465f832021525894519f97" amd64: cloud-hypervisor: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/cloud-hypervisor" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/amd64/cloud-hypervisor" sha256: "448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc" virtiofsd: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/virtiofsd" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/amd64/virtiofsd" sha256: "15b2e72a78cc08a9bd8a6943e89fb69c88cb3cbeb63069efceade835342ac7d4" kata-kernel: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/vmlinux" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/amd64/vmlinux" sha256: "f87d8359035cb7557b16a319e6431ea09b183e62268aef436bf8dc6530d786b6" kata-image: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/rootfs.img" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/amd64/rootfs.img" sha256: "6770db2cca5710593f27c9ac5e1d9d7aab836422695a5e5c4807496a92a0f24d" From 908ff98118e718f7d8d9f828b4e6405079b7bf83 Mon Sep 17 00:00:00 2001 From: Noah West Date: Sun, 18 Oct 2026 17:48:18 -0700 Subject: [PATCH 05/10] ate-setup: cache multi-arch micro-VM assets and verify digests before stamping Cache the full bin/microvm-assets directory (both amd64 and arm64) in the PR workflow instead of only bin/microvm-assets/amd64. Verify SHA-256 digests inside assembleMicroVMAssets before writing .asset-versions so a corrupt download is never stamped, and drop the redundant virtiofsd --version subprocess check. Move sandboxconfig-microvm.yaml.tmpl from the single-file manifests/microvm directory into manifests/ate-install alongside the other ate-setup manifests. Signed-off-by: Noah West --- .github/workflows/pr-workflow.yaml | 15 +++---- .../ate/v1/sandboxconfig_validation_test.go | 2 +- cmd/ate-setup/commands.md | 2 +- cmd/ate-setup/differences.md | 6 +-- cmd/ate-setup/internal/steps/microvm.go | 44 +++++-------------- cmd/ate-setup/internal/steps/microvm_test.go | 25 +++++++---- docs/dev/microvm-local.md | 2 +- .../sandboxconfig-microvm.yaml.tmpl | 0 8 files changed, 41 insertions(+), 55 deletions(-) rename manifests/{microvm => ate-install}/sandboxconfig-microvm.yaml.tmpl (100%) diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 61d6115a81..70a43db96b 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -258,14 +258,13 @@ jobs: id: microvm-assets uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - # Assembling the assets (download kata-static + cloud-hypervisor + the - # prebuilt virtiofsd — all downloads on amd64) is fully pinned by - # cmd/ate-setup/internal/steps/microvm.go, so key the cache on its hash. - # The push-to-main run populates the default-branch cache PRs inherit; - # the weekly schedule refreshes it. - # run-microvm-demo-kind.sh skips assembling when these are present. - path: bin/microvm-assets/amd64 - key: microvm-assets-amd64-${{ hashFiles('cmd/ate-setup/internal/steps/microvm.go') }} + # Assembling the assets (download kata-static + cloud-hypervisor for + # amd64 and arm64) is fully pinned by cmd/ate-setup/internal/steps/microvm.go, + # so key the cache on its hash. The push-to-main run populates the + # default-branch cache PRs inherit; the weekly schedule refreshes it. + # ate-setup skips assembling when these are present. + path: bin/microvm-assets + key: microvm-assets-${{ hashFiles('cmd/ate-setup/internal/steps/microvm.go') }} - name: Enable KVM # Grant the runner access to /dev/kvm so create-kind-cluster.sh mounts it into # the node and labels it for the micro-VM sandbox class. diff --git a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go index 6b6663cec9..a05a783020 100644 --- a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go +++ b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go @@ -352,7 +352,7 @@ func TestShippedSandboxConfigManifests(t *testing.T) { for _, path := range []string{ "../../../../../manifests/ate-install/sandboxconfig-gvisor.yaml", - "../../../../../manifests/microvm/sandboxconfig-microvm.yaml.tmpl", + "../../../../../manifests/ate-install/sandboxconfig-microvm.yaml.tmpl", } { t.Run(path, func(t *testing.T) { raw, err := os.ReadFile(path) diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index d7f58e544a..67fce5fdd5 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -109,7 +109,7 @@ that already names a manifest is used as written, and is not looked up. |---|---| | `deploy ate-system` | `--deploy-ate-system` | | `deploy ate-system --setup-csi=nfs` | `--deploy-ate-system --setup-csi=nfs` | -| `deploy ate-system --skip-microvm-deps` | `--deploy-ate-system --skip-microvm-deps` | +| `deploy ate-system --skip-microvm-deps` | (no shell equivalent) | | `deploy atelet` | `--deploy-atelet` | | `deploy apiserver` | `--deploy-ate-apiserver` | | `deploy ate-controller` | (no shell equivalent) | diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index cfbdd12268..d104de49cd 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -5,10 +5,8 @@ `hack/install-microvm-deps.sh`. The port has replaced them. The demo, CSI, and micro-VM asset scripts are -deleted, and `hack/install-ate.sh` is a shim holding no install logic of its -own: it translates the flags and environment variables the installer has always -accepted onto `ate-setup` commands, so existing command lines keep working. See -[`commands.md`](commands.md) for the flag-by-flag mapping. +deleted, and `hack/install-ate.sh` forwards its arguments directly to +`ate-setup`. See [`commands.md`](commands.md) for the historical flag mapping. This document covers what changed *behind* that mapping: the design decisions of the port, against the shell it replaced. For what a user of diff --git a/cmd/ate-setup/internal/steps/microvm.go b/cmd/ate-setup/internal/steps/microvm.go index def655a719..5257bd1b13 100644 --- a/cmd/ate-setup/internal/steps/microvm.go +++ b/cmd/ate-setup/internal/steps/microvm.go @@ -51,13 +51,10 @@ const ( defaultKataVersion = "4.1.0" defaultCloudHypervisorVersion = "v53.0" - // virtiofsdVersion is the version bundled inside kata-static for - // defaultKataVersion (verified before stamping). - virtiofsdVersion = "1.14.0" assetStampFile = ".asset-versions" defaultKindMicroVMBucket = "ate-snapshots" - microvmSandboxConfigTemplate = "manifests/microvm/sandboxconfig-microvm.yaml.tmpl" + microvmSandboxConfigTemplate = "manifests/ate-install/sandboxconfig-microvm.yaml.tmpl" microvmSandboxConfigName = "microvm" // Keep in sync with manifests/ate-install/kind/rustfs.yaml. @@ -76,7 +73,7 @@ var ( microvmAssets = []string{"cloud-hypervisor", "virtiofsd", "vmlinux", "rootfs.img"} // microvmAssetDigests pins the expected sha256 digest of each assembled - // file in microvmAssets, matching manifests/microvm/sandboxconfig-microvm.yaml.tmpl. + // file in microvmAssets, matching manifests/ate-install/sandboxconfig-microvm.yaml.tmpl. microvmAssetDigests = map[string]map[string]string{ "arm64": { "cloud-hypervisor": "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850", @@ -129,9 +126,9 @@ func (e *Env) DeployMicroVMDeps(ctx context.Context) error { } } else { log.Infof("Assets already present in %s; skipping assemble.", outDir) - } - if err := verifyMicroVMAssetDigests(outDir, arch); err != nil { - return err + if err := verifyMicroVMAssetDigests(outDir, arch); err != nil { + return err + } } } @@ -149,7 +146,7 @@ func (e *Env) DeployMicroVMDeps(ctx context.Context) error { log.Infof("Applying microvm SandboxConfig from %s...", microvmSandboxConfigTemplate) rendered, err := render.Template( - e.Cfg.Path("manifests", "microvm", "sandboxconfig-microvm.yaml.tmpl"), + e.Cfg.Manifest("sandboxconfig-microvm.yaml.tmpl"), map[string]string{ "BUCKET_SCHEME": e.microVMBucketScheme(), "ASSET_BUCKET_NAME": bucket, @@ -162,7 +159,7 @@ func (e *Env) DeployMicroVMDeps(ctx context.Context) error { if err := e.Kube.ApplyBytes(ctx, rendered); err != nil { return err } - log.Infof("Done. ActorTemplates must reference this SandboxConfig by name (sandboxConfig.configName: microvm).") + log.Infof("Done. Installed cluster-wide microvm SandboxConfig.") return nil } @@ -226,8 +223,8 @@ func cloudHypervisorAssetName(arch string) (string, error) { } func microvmAssetStamp(arch, kataVer, chVer string) string { - return fmt.Sprintf("arch=%s\nkata=%s\ncloud-hypervisor=%s\nvirtiofsd=%s\n", - arch, kataVer, chVer, virtiofsdVersion) + return fmt.Sprintf("arch=%s\nkata=%s\ncloud-hypervisor=%s\n", + arch, kataVer, chVer) } func checkMicroVMAssets(outDir string) error { @@ -282,8 +279,8 @@ func microvmAssetsNeedAssemble(outDir, wantStamp string) (needAssemble, stale bo } // assembleMicroVMAssets downloads the Kata static tarball and Cloud Hypervisor -// binary for arch into outDir, verifies the bundled virtiofsd version when -// runnable on the host, and writes the .asset-versions stamp last. +// binary for arch into outDir, verifies the SHA-256 digests against +// microvmAssetDigests, and writes the .asset-versions stamp last. func assembleMicroVMAssets(ctx context.Context, outDir, arch, kataVer, chVer string) error { chAsset, err := cloudHypervisorAssetName(arch) if err != nil { @@ -322,7 +319,7 @@ func assembleMicroVMAssets(ctx context.Context, outDir, arch, kataVer, chVer str if err := checkMicroVMAssets(outDir); err != nil { return err } - if err := verifyVirtiofsdVersion(ctx, filepath.Join(outDir, "virtiofsd"), kataVer, virtiofsdVersion); err != nil { + if err := verifyMicroVMAssetDigests(outDir, arch); err != nil { return err } if err := os.WriteFile(stampPath, []byte(microvmAssetStamp(arch, kataVer, chVer)), 0o644); err != nil { @@ -492,23 +489,6 @@ func downloadToFile(ctx context.Context, url, dstPath string, mode fs.FileMode) return writeStreamToFile(dstPath, mode, resp.Body) } -func verifyVirtiofsdVersion(ctx context.Context, virtiofsdPath, kataVer, wantVersion string) error { - out, err := exec.CommandContext(ctx, virtiofsdPath, "--version").Output() - if err != nil { - return ctx.Err() - } - firstLine, _, _ := strings.Cut(string(out), "\n") - fields := strings.Fields(firstLine) - if len(fields) < 2 { - return nil - } - if got := fields[1]; got != wantVersion { - return fmt.Errorf("kata %s bundles virtiofsd %s, not %s: update virtiofsdVersion", kataVer, got, wantVersion) - } - log.Infof("virtiofsd %s", fields[1]) - return nil -} - func (e *Env) stageMicroVMAssetsToRustfs(ctx context.Context, bucket string) error { if _, err := exec.LookPath("docker"); err != nil { return fmt.Errorf("'docker' is required to run the S3 client but was not found in PATH: %w", err) diff --git a/cmd/ate-setup/internal/steps/microvm_test.go b/cmd/ate-setup/internal/steps/microvm_test.go index d0092d54cf..e5701ea149 100644 --- a/cmd/ate-setup/internal/steps/microvm_test.go +++ b/cmd/ate-setup/internal/steps/microvm_test.go @@ -116,7 +116,7 @@ func TestVerifyMicroVMAssetDigests(t *testing.T) { func TestMicroVMAssetsNeedAssemble(t *testing.T) { stamp := microvmAssetStamp("amd64", defaultKataVersion, defaultCloudHypervisorVersion) - const wantStamp = "arch=amd64\nkata=4.1.0\ncloud-hypervisor=v53.0\nvirtiofsd=1.14.0\n" + const wantStamp = "arch=amd64\nkata=4.1.0\ncloud-hypervisor=v53.0\n" if stamp != wantStamp { t.Fatalf("microvmAssetStamp() =\n%q\nwant\n%q", stamp, wantStamp) } @@ -288,8 +288,17 @@ func serveFakeReleases(t *testing.T, archive []byte, chBody string, failCH bool) func TestAssembleMicroVMAssets(t *testing.T) { silenceStepLog(t) - virtiofsdBody := "#!/bin/sh\necho 'virtiofsd " + virtiofsdVersion + "'\n" + const virtiofsdBody = "fake-virtiofsd-binary" const chBody = "fake-cloud-hypervisor-binary" + origDigests := microvmAssetDigests["amd64"] + microvmAssetDigests["amd64"] = map[string]string{ + "cloud-hypervisor": sha256HexString(chBody), + "virtiofsd": sha256HexString(virtiofsdBody), + "vmlinux": sha256HexString("fake-kernel-bytes"), + "rootfs.img": sha256HexString("fake-rootfs-bytes"), + } + t.Cleanup(func() { microvmAssetDigests["amd64"] = origDigests }) + serveFakeReleases(t, validKataArchive(t, virtiofsdBody), chBody, false) outDir := t.TempDir() @@ -334,7 +343,7 @@ func TestAssembleMicroVMAssets(t *testing.T) { func TestAssembleMicroVMAssetsClearsStampOnFailure(t *testing.T) { silenceStepLog(t) - serveFakeReleases(t, validKataArchive(t, "#!/bin/sh\necho 'virtiofsd "+virtiofsdVersion+"'\n"), "", true) + serveFakeReleases(t, validKataArchive(t, "fake-virtiofsd-binary"), "", true) outDir := t.TempDir() stampPath := filepath.Join(outDir, assetStampFile) @@ -350,18 +359,18 @@ func TestAssembleMicroVMAssetsClearsStampOnFailure(t *testing.T) { } } -func TestAssembleMicroVMAssetsRejectsVirtiofsdVersionMismatch(t *testing.T) { +func TestAssembleMicroVMAssetsRejectsDigestMismatchBeforeStamping(t *testing.T) { silenceStepLog(t) - serveFakeReleases(t, validKataArchive(t, "#!/bin/sh\necho 'virtiofsd 1.13.3'\n"), "ch", false) + serveFakeReleases(t, validKataArchive(t, "bad-virtiofsd-bytes"), "ch", false) outDir := t.TempDir() err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.1.0", "v53.0") - if err == nil || !strings.Contains(err.Error(), "1.13.3") { - t.Fatalf("assembleMicroVMAssets() = %v, want virtiofsd 1.13.3 mismatch error", err) + if err == nil || !strings.Contains(err.Error(), "sha256 =") { + t.Fatalf("assembleMicroVMAssets() = %v, want sha256 mismatch error", err) } if _, statErr := os.Stat(filepath.Join(outDir, assetStampFile)); !errors.Is(statErr, os.ErrNotExist) { - t.Errorf("stamp written despite virtiofsd mismatch: %v", statErr) + t.Errorf("stamp written despite digest mismatch: %v", statErr) } } diff --git a/docs/dev/microvm-local.md b/docs/dev/microvm-local.md index 8c09b13eca..9c533da88e 100644 --- a/docs/dev/microvm-local.md +++ b/docs/dev/microvm-local.md @@ -28,7 +28,7 @@ asset set is four files per architecture (`amd64` and `arm64`): `go run ./cmd/ate-setup deploy ate-system` (and `deploy sandboxconfig`) assembles both architectures into `bin/microvm-assets/{amd64,arm64}/`, verifies -their SHA-256 digests against `manifests/microvm/sandboxconfig-microvm.yaml.tmpl`, +their SHA-256 digests against `manifests/ate-install/sandboxconfig-microvm.yaml.tmpl`, stages them into the cluster's object store under `kata-assets//` (the shared `rustfs` bucket on kind, and `-assets` on GKE, the only bucket `setup-gcp` lets nodes read), and applies the cluster-wide `microvm` diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl similarity index 100% rename from manifests/microvm/sandboxconfig-microvm.yaml.tmpl rename to manifests/ate-install/sandboxconfig-microvm.yaml.tmpl From 36fa1a36363cf8f42cc2e81f9d2d478faa5cc3b4 Mon Sep 17 00:00:00 2001 From: Noah West Date: Tue, 20 Oct 2026 17:06:53 -0700 Subject: [PATCH 06/10] ate-setup: stage micro-VM assets under microvm-assets// Stage assembled micro-VM guest assets under microvm-assets/v1// in the object store and bin/microvm-assets/v1// locally instead of unversioned kata-assets//. Versioned paths prevent future SandboxConfig bumps from overwriting assets needed to restore suspended actors. Signed-off-by: Noah West --- cmd/ate-setup/internal/steps/microvm.go | 27 ++++++----- cmd/ate-setup/internal/steps/microvm_test.go | 46 ++++++++++++++++++- docs/dev/microvm-local.md | 10 ++-- .../sandboxconfig-microvm.yaml.tmpl | 23 +++++----- 4 files changed, 77 insertions(+), 29 deletions(-) diff --git a/cmd/ate-setup/internal/steps/microvm.go b/cmd/ate-setup/internal/steps/microvm.go index 5257bd1b13..472b7264c2 100644 --- a/cmd/ate-setup/internal/steps/microvm.go +++ b/cmd/ate-setup/internal/steps/microvm.go @@ -49,11 +49,13 @@ const ( // and bucket-init Job on Kind. MicroVMStageTimeout = 300 * time.Second + defaultMicroVMAssetVersion = "v1" defaultKataVersion = "4.1.0" defaultCloudHypervisorVersion = "v53.0" assetStampFile = ".asset-versions" defaultKindMicroVMBucket = "ate-snapshots" + microvmAssetsPrefix = "microvm-assets" microvmSandboxConfigTemplate = "manifests/ate-install/sandboxconfig-microvm.yaml.tmpl" microvmSandboxConfigName = "microvm" @@ -113,7 +115,7 @@ func (e *Env) DeployMicroVMDeps(ctx context.Context) error { } for _, arch := range microvmArches { - outDir := e.Cfg.Path("bin", "microvm-assets", arch) + outDir := e.Cfg.Path("bin", microvmAssetsPrefix, defaultMicroVMAssetVersion, arch) wantStamp := microvmAssetStamp(arch, defaultKataVersion, defaultCloudHypervisorVersion) needAssemble, stale := microvmAssetsNeedAssemble(outDir, wantStamp) if stale { @@ -133,12 +135,12 @@ func (e *Env) DeployMicroVMDeps(ctx context.Context) error { } if e.Cfg.Kind { - log.Infof("Staging assets to in-cluster rustfs bucket %s (kata-assets/)...", bucket) + log.Infof("Staging assets to in-cluster rustfs bucket %s (%s/%s/)...", bucket, microvmAssetsPrefix, defaultMicroVMAssetVersion) if err := e.stageMicroVMAssetsToRustfs(ctx, bucket); err != nil { return err } } else { - log.Infof("Uploading assets to gs://%s/kata-assets/ ...", bucket) + log.Infof("Uploading assets to gs://%s/%s/%s/ ...", bucket, microvmAssetsPrefix, defaultMicroVMAssetVersion) if err := e.stageMicroVMAssetsToGCS(ctx, bucket); err != nil { return err } @@ -179,7 +181,7 @@ func (e *Env) DeleteMicroVMDeps(ctx context.Context) error { return err } if bucket, err := e.microVMBucket(); err == nil { - log.Infof("Done. (Bucket assets at %s://%s/kata-assets/ left in place.)", e.microVMBucketScheme(), bucket) + log.Infof("Done. (Bucket assets at %s://%s/%s/ left in place.)", e.microVMBucketScheme(), bucket, microvmAssetsPrefix) } else { log.Infof("Done.") } @@ -531,16 +533,17 @@ func (e *Env) stageMicroVMAssetsToRustfs(ctx context.Context, bucket string) err return e.runCmd(ctx, stdin, "docker", dockerArgs...) } - log.Infof(">> Uploading assets to s3://%s/kata-assets/ via %s (netns of %s)...", bucket, endpoint, node) + log.Infof(">> Uploading assets to s3://%s/%s/%s/ via %s (netns of %s)...", bucket, microvmAssetsPrefix, defaultMicroVMAssetVersion, endpoint, node) for _, arch := range microvmArches { - outDir := e.Cfg.Path("bin", "microvm-assets", arch) + outDir := e.Cfg.Path("bin", microvmAssetsPrefix, defaultMicroVMAssetVersion, arch) for _, f := range microvmAssets { log.Infof(" %s/%s", arch, f) file, err := os.Open(filepath.Join(outDir, f)) if err != nil { return err } - err = awsCLI(file, "s3", "cp", "-", fmt.Sprintf("s3://%s/kata-assets/%s/%s", bucket, arch, f)) + dst := fmt.Sprintf("s3://%s/%s/%s/%s/%s", bucket, microvmAssetsPrefix, defaultMicroVMAssetVersion, arch, f) + err = awsCLI(file, "s3", "cp", "-", dst) _ = file.Close() if err != nil { return fmt.Errorf("while uploading %s/%s to rustfs: %w", arch, f, err) @@ -549,7 +552,7 @@ func (e *Env) stageMicroVMAssetsToRustfs(ctx context.Context, bucket string) err } log.Infof(">> Done. Verify:") - return awsCLI(nil, "s3", "ls", "--recursive", fmt.Sprintf("s3://%s/kata-assets/", bucket)) + return awsCLI(nil, "s3", "ls", "--recursive", fmt.Sprintf("s3://%s/%s/", bucket, microvmAssetsPrefix)) } func (e *Env) waitJobComplete(ctx context.Context, namespace, name string, timeout time.Duration) error { @@ -581,11 +584,11 @@ func (e *Env) stageMicroVMAssetsToGCS(ctx context.Context, bucket string) error } for _, arch := range microvmArches { - outDir := e.Cfg.Path("bin", "microvm-assets", arch) + outDir := e.Cfg.Path("bin", microvmAssetsPrefix, defaultMicroVMAssetVersion, arch) for _, f := range microvmAssets { log.Infof(" %s/%s", arch, f) - args := append(append([]string{"storage", "cp"}, projectFlag...), - filepath.Join(outDir, f), fmt.Sprintf("gs://%s/kata-assets/%s/%s", bucket, arch, f)) + dst := fmt.Sprintf("gs://%s/%s/%s/%s/%s", bucket, microvmAssetsPrefix, defaultMicroVMAssetVersion, arch, f) + args := append(append([]string{"storage", "cp"}, projectFlag...), filepath.Join(outDir, f), dst) if err := e.runCmd(ctx, nil, "gcloud", args...); err != nil { return fmt.Errorf("while uploading %s/%s to GCS: %w", arch, f, err) } @@ -593,7 +596,7 @@ func (e *Env) stageMicroVMAssetsToGCS(ctx context.Context, bucket string) error } log.Infof(">> Done. Verify:") - lsArgs := append(append([]string{"storage", "ls", "--recursive"}, projectFlag...), fmt.Sprintf("gs://%s/kata-assets/", bucket)) + lsArgs := append(append([]string{"storage", "ls", "--recursive"}, projectFlag...), fmt.Sprintf("gs://%s/%s/", bucket, microvmAssetsPrefix)) return e.runCmd(ctx, nil, "gcloud", lsArgs...) } diff --git a/cmd/ate-setup/internal/steps/microvm_test.go b/cmd/ate-setup/internal/steps/microvm_test.go index e5701ea149..2a3ea3b8ec 100644 --- a/cmd/ate-setup/internal/steps/microvm_test.go +++ b/cmd/ate-setup/internal/steps/microvm_test.go @@ -436,7 +436,7 @@ func TestMicroVMSandboxConfigTemplate(t *testing.T) { } for _, arch := range microvmArches { for _, asset := range microvmAssets { - wantURL := tc.scheme + "://" + tc.bucket + "/kata-assets/" + arch + "/" + asset + wantURL := tc.scheme + "://" + tc.bucket + "/" + microvmAssetsPrefix + "/" + defaultMicroVMAssetVersion + "/" + arch + "/" + asset if !strings.Contains(string(rendered), wantURL) { t.Errorf("rendered manifest missing expected bucket URL %q:\n%s", wantURL, rendered) } @@ -457,6 +457,50 @@ func TestMicroVMSandboxConfigTemplate(t *testing.T) { } } +func TestStageMicroVMAssetsToGCS(t *testing.T) { + silenceStepLog(t) + + root := t.TempDir() + binDir := t.TempDir() + callsFile := filepath.Join(t.TempDir(), "gcloud-calls.txt") + if err := os.WriteFile(filepath.Join(binDir, "gcloud"), []byte("#!/bin/sh\necho \"$*\" >> "+callsFile+"\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + for _, arch := range microvmArches { + dir := filepath.Join(root, "bin", microvmAssetsPrefix, defaultMicroVMAssetVersion, arch) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for _, f := range microvmAssets { + if err := os.WriteFile(filepath.Join(dir, f), []byte(f), 0o644); err != nil { + t.Fatal(err) + } + } + } + + env := &Env{Cfg: &config.Config{Root: root, BucketName: "test-bucket"}} + if err := env.stageMicroVMAssetsToGCS(t.Context(), "test-bucket-assets"); err != nil { + t.Fatalf("stageMicroVMAssetsToGCS() = %v", err) + } + + gotBytes, err := os.ReadFile(callsFile) + if err != nil { + t.Fatal(err) + } + got := string(gotBytes) + for _, arch := range microvmArches { + for _, f := range microvmAssets { + src := filepath.Join(root, "bin", microvmAssetsPrefix, defaultMicroVMAssetVersion, arch, f) + dst := "gs://test-bucket-assets/" + microvmAssetsPrefix + "/" + defaultMicroVMAssetVersion + "/" + arch + "/" + f + if want := "storage cp " + src + " " + dst; !strings.Contains(got, want) { + t.Errorf("missing expected gcloud upload %q in:\n%s", want, got) + } + } + } +} + func TestDeployMicroVMDepsRequiresBucketOnNonKind(t *testing.T) { silenceStepLog(t) diff --git a/docs/dev/microvm-local.md b/docs/dev/microvm-local.md index 9c533da88e..d03512362e 100644 --- a/docs/dev/microvm-local.md +++ b/docs/dev/microvm-local.md @@ -27,10 +27,10 @@ asset set is four files per architecture (`amd64` and `arm64`): - `rootfs.img` — the guest rootfs image (from `kata-static`) `go run ./cmd/ate-setup deploy ate-system` (and `deploy sandboxconfig`) -assembles both architectures into `bin/microvm-assets/{amd64,arm64}/`, verifies +assembles both architectures into `bin/microvm-assets/v1/{amd64,arm64}/`, verifies their SHA-256 digests against `manifests/ate-install/sandboxconfig-microvm.yaml.tmpl`, -stages them into the cluster's object store under `kata-assets//` (the -shared `rustfs` bucket on kind, and `-assets` on GKE, the only +stages them into the cluster's object store under `microvm-assets///` +(the shared `rustfs` bucket on kind, and `-assets` on GKE, the only bucket `setup-gcp` lets nodes read), and applies the cluster-wide `microvm` `SandboxConfig`. Assets must not be stored under a snapshot location (an ActorTemplate's `snapshotConfig.location` followed by `/atespaces/`), as the @@ -179,8 +179,8 @@ cd ``` The first run of `run-microvm-demo-kind.sh` downloads the guest runtime -releases and extracts them into `bin/microvm-assets/{amd64,arm64}/`; later runs -reuse those directories when the version stamps and SHA-256 digests match. +releases and extracts them into `bin/microvm-assets/v1/{amd64,arm64}/`; later +runs reuse those directories when the version stamps and SHA-256 digests match. Verify as in Option A, step 4. diff --git a/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl b/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl index c928992790..b302788733 100644 --- a/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl @@ -21,8 +21,9 @@ # # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs) # are FETCHED at runtime from the asset bucket ${ASSET_BUCKET_NAME} under -# kata-assets// (the rustfs bucket on kind, its own GCS bucket on GKE), -# through the node's object-store plugin - NOT baked into the worker image. +# microvm-assets/// (the rustfs bucket on kind, its own GCS +# bucket on GKE), through the node's object-store plugin - NOT baked into the +# worker image. # ${BUCKET_SCHEME} is that store's URL scheme (s3 on kind, gs on GKE): the # plugin serves only its own backend's scheme. The assets must stay outside any # snapshot location, which the plugin refuses to serve as assets. ateworker @@ -32,7 +33,7 @@ # The per-arch sha256 values below are the asset sets assembled and verified by # cmd/ate-setup/internal/steps/microvm.go; atelet selects the block matching # the node's architecture, and ate-setup stages both architectures' binaries at -# these paths. +# these versioned paths so newer releases do not overwrite kept versions. apiVersion: ate.dev/v1 kind: SandboxConfig @@ -49,28 +50,28 @@ spec: assets: arm64: cloud-hypervisor: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/arm64/cloud-hypervisor" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/v2/arm64/cloud-hypervisor" sha256: "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850" # virtiofsd serves the overlay RO lower (virtio-fs). virtiofsd: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/arm64/virtiofsd" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/v2/arm64/virtiofsd" sha256: "4b43f4f6de6500203b01214a790314eea4182f5f110594ddb28ae2805622f06f" kata-kernel: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/arm64/vmlinux" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/v2/arm64/vmlinux" sha256: "dece388830720fe8bddb41552fff5f486240aeece5acc0aebc1812761e3e3f66" kata-image: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/arm64/rootfs.img" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/v2/arm64/rootfs.img" sha256: "69f9a76e5a2563b7c74d1f271f3a904240f50ef405465f832021525894519f97" amd64: cloud-hypervisor: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/amd64/cloud-hypervisor" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/v2/amd64/cloud-hypervisor" sha256: "448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc" virtiofsd: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/amd64/virtiofsd" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/v2/amd64/virtiofsd" sha256: "15b2e72a78cc08a9bd8a6943e89fb69c88cb3cbeb63069efceade835342ac7d4" kata-kernel: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/amd64/vmlinux" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/v2/amd64/vmlinux" sha256: "f87d8359035cb7557b16a319e6431ea09b183e62268aef436bf8dc6530d786b6" kata-image: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/amd64/rootfs.img" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/v2/amd64/rootfs.img" sha256: "6770db2cca5710593f27c9ac5e1d9d7aab836422695a5e5c4807496a92a0f24d" From 54fe9a3a27d98f10ad38915c769b00f4822a562a Mon Sep 17 00:00:00 2001 From: Noah West Date: Fri, 23 Oct 2026 09:44:56 -0700 Subject: [PATCH 07/10] ate-setup: align microvm-deps with ate.dev/v1 and asset bucket DeleteMicroVMDeps still targeted ate.dev/v1alpha1 after CRDs moved to ate.dev/v1, which kube.Client.DeleteOne ignored as a no-match error. Use atev1.GroupVersion.WithKind("SandboxConfig") instead. Reject BUCKET_NAME values whose -assets bucket exceeds GCS's 63-character limit, matching setup-gcp's checkAssetBucketName. Pass --skip-microvm-deps from orchestrator.substrate_args when sandboxClass is not microvm so gVisor benchmark runs do not need write access to -assets. Signed-off-by: Noah West --- benchmarking/automation/README.md | 7 +-- benchmarking/automation/orchestrator.py | 13 +++-- benchmarking/automation/test_orchestrator.py | 20 ++++++-- cmd/ate-setup/commands.md | 8 +-- cmd/ate-setup/differences.md | 9 ++-- cmd/ate-setup/internal/steps/microvm.go | 17 +++---- cmd/ate-setup/internal/steps/microvm_test.go | 51 +++++++++++++++++++- 7 files changed, 95 insertions(+), 30 deletions(-) diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 27f73f65ec..b84b9f662a 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -20,9 +20,10 @@ the router capacity benchmark — see and/or `${KO_DOCKER_REPO}/nighthawk-ingress-test:`. 5. `hack/install-ate.sh deploy ate-system` (with `ATE_CREDENTIAL_PROVIDER` in `.ate-dev-env.sh`, or `--credential-provider` in the test's - `ateArgs`, which also stages the micro-VM guest assets to the cluster's - object store bucket and applies both the `gvisor-default` and `microvm` - `SandboxConfig`s) + `benchmarking/workloads/deploy.sh + `ateArgs`; passes `--skip-microvm-deps` unless `sandboxClass` is `microvm`, + in which case `deploy ate-system` also stages the micro-VM guest assets to + `-assets` and applies the cluster-wide `microvm` + `SandboxConfig`) + `benchmarking/workloads/deploy.sh --deploy --sandbox-class ` (these build & push substrate / workload images via `ko` as part of their deploy steps — there's no separate image build step). The install sets `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` to diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index 2eb06df4e5..4e6bca1a71 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -391,10 +391,15 @@ def validate_and_normalize_tests(tests: list[dict[str, Any]]) -> None: def substrate_args(test: dict[str, Any]) -> list[str]: """Return the hack/install-ate.sh deploy ate-system arguments for a - tests.yaml entry: its ateArgs, then the fake data plane flag its isolate - block asks for. The teardown needs no matching flag: delete ate-system - removes the fake data plane whether or not it was installed.""" - return [*test.get("ateArgs", []), *isolate_install_args(test)] + tests.yaml entry: its ateArgs, the fake data plane flag its isolate block + asks for, and --skip-microvm-deps when sandboxClass is not microvm so + gVisor runs do not need write access to -assets. The teardown + needs no matching flag: delete ate-system removes the fake data plane and + any microvm SandboxConfig whether or not they were installed.""" + args = [*test.get("ateArgs", []), *isolate_install_args(test)] + if test.get("sandboxClass", "gvisor") != "microvm" and "--skip-microvm-deps" not in args: + args.append("--skip-microvm-deps") + return args # ate-api-server enforces authorization, and the runner Job calls it with its diff --git a/benchmarking/automation/test_orchestrator.py b/benchmarking/automation/test_orchestrator.py index 77833d43ba..ec4dd904b4 100644 --- a/benchmarking/automation/test_orchestrator.py +++ b/benchmarking/automation/test_orchestrator.py @@ -40,7 +40,7 @@ class SubstrateArgsTest(unittest.TestCase): def test_ate_args_only(self): self.assertEqual( orchestrator.substrate_args({"ateArgs": ["--cluster-size=size10"]}), - ["--cluster-size=size10"], + ["--cluster-size=size10", "--skip-microvm-deps"], ) def test_isolate_installs_the_fake_data_plane(self): @@ -50,14 +50,26 @@ def test_isolate_installs_the_fake_data_plane(self): "isolate": {"delay": "500ms"}, }), # The delay tunes fake-atelet through deploy.sh, not the install. - ["--cluster-size=size10", "--fake-data-plane"], + ["--cluster-size=size10", "--fake-data-plane", "--skip-microvm-deps"], ) def test_neither(self): - self.assertEqual(orchestrator.substrate_args({}), []) + self.assertEqual(orchestrator.substrate_args({}), ["--skip-microvm-deps"]) def test_bare_isolate_key(self): - self.assertEqual(orchestrator.substrate_args({"isolate": None}), ["--fake-data-plane"]) + self.assertEqual( + orchestrator.substrate_args({"isolate": None}), + ["--fake-data-plane", "--skip-microvm-deps"], + ) + + def test_microvm_keeps_microvm_deps(self): + self.assertEqual( + orchestrator.substrate_args({ + "sandboxClass": "microvm", + "ateArgs": ["--cluster-size=size10"], + }), + ["--cluster-size=size10"], + ) class DeploySubstrateTest(unittest.TestCase): diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 67fce5fdd5..1e6c6f4728 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -121,10 +121,10 @@ that already names a manifest is used as written, and is not looked up. `deploy ate-system` is the whole control plane: CRDs, RBAC, the store, the apiserver, the controller, atenet, and atelet. Unless `--skip-microvm-deps` is passed (also accepted by `deploy sandboxconfig`), it also stages the micro-VM -guest assets into the cluster object store and applies the `microvm` -cluster-wide `SandboxConfig` alongside `gvisor-default`. It creates every -`create` resource below on the way, so those subcommands are only needed to -redo one on a running cluster. +guest assets into the asset bucket (`rustfs` on Kind, `-assets` on +GKE) and applies the `microvm` cluster-wide `SandboxConfig` alongside +`gvisor-default`. It creates every `create` resource below on the way, so those +subcommands are only needed to redo one on a running cluster. ## Publish diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index d104de49cd..20cc17e695 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -95,10 +95,11 @@ reconstructs the environment the shell installer would have exported to it, and is fronted by `deploy`/`delete benchmarks`. The micro-VM asset assembly (`kata-static` `.tar.zst` extraction and -`cloud-hypervisor` download), object-store staging (`rustfs` on Kind, GCS on -GKE), and `microvm` `SandboxConfig` lifecycle now run in Go as part of `deploy -ate-system` (and `deploy sandboxconfig`), with `delete ate-system` removing the -`microvm` `SandboxConfig` alongside the rest of the control plane. +`cloud-hypervisor` download), object-store staging (`rustfs` on Kind, +`-assets` on GKE), and `microvm` `SandboxConfig` lifecycle now run +in Go as part of `deploy ate-system` (and `deploy sandboxconfig`), with +`delete ate-system` removing the `microvm` `SandboxConfig` alongside the rest +of the control plane. `ko` is no longer asked to apply anything. The scripts ran `run_ko apply`, which made ko shell out to kubectl and forced the awkward `-- --context=` special case diff --git a/cmd/ate-setup/internal/steps/microvm.go b/cmd/ate-setup/internal/steps/microvm.go index 472b7264c2..0c506b1bc6 100644 --- a/cmd/ate-setup/internal/steps/microvm.go +++ b/cmd/ate-setup/internal/steps/microvm.go @@ -36,9 +36,9 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" + atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/cmd/ate-setup/internal/log" "go.ate.dev/substrate/cmd/ate-setup/internal/render" @@ -55,6 +55,7 @@ const ( assetStampFile = ".asset-versions" defaultKindMicroVMBucket = "ate-snapshots" + maxGCSBucketNameLength = 63 microvmAssetsPrefix = "microvm-assets" microvmSandboxConfigTemplate = "manifests/ate-install/sandboxconfig-microvm.yaml.tmpl" microvmSandboxConfigName = "microvm" @@ -92,12 +93,6 @@ var ( } ) -var sandboxConfigGVK = schema.GroupVersionKind{ - Group: "ate.dev", - Version: "v1alpha1", - Kind: "SandboxConfig", -} - // microvmReleaseBaseURL is overridden in unit tests to point at an httptest.Server. var microvmReleaseBaseURL = "https://github.com" @@ -175,7 +170,7 @@ func (e *Env) DeleteMicroVMDeps(ctx context.Context) error { log.Infof("Deleting microvm SandboxConfig...") obj := &unstructured.Unstructured{} - obj.SetGroupVersionKind(sandboxConfigGVK) + obj.SetGroupVersionKind(atev1.GroupVersion.WithKind("SandboxConfig")) obj.SetName(microvmSandboxConfigName) if err := e.Kube.DeleteOne(ctx, obj); err != nil { return err @@ -201,7 +196,11 @@ func (e *Env) microVMBucket() (string, error) { if e.Cfg.BucketName == "" { return "", fmt.Errorf("BUCKET_NAME (--storage-bucket-name or config.storage.bucketName) must be set to stage micro-VM assets on non-Kind clusters, or pass --skip-microvm-deps") } - return e.Cfg.BucketName + "-assets", nil + assetBucket := e.Cfg.BucketName + "-assets" + if len(assetBucket) > maxGCSBucketNameLength { + return "", fmt.Errorf("bucket name %q is too long: the asset bucket %q would have %d characters, and GCS allows %d", e.Cfg.BucketName, assetBucket, len(assetBucket), maxGCSBucketNameLength) + } + return assetBucket, nil } // microVMBucketScheme returns the URL scheme for the node's object-store diff --git a/cmd/ate-setup/internal/steps/microvm_test.go b/cmd/ate-setup/internal/steps/microvm_test.go index 2a3ea3b8ec..9488ac21df 100644 --- a/cmd/ate-setup/internal/steps/microvm_test.go +++ b/cmd/ate-setup/internal/steps/microvm_test.go @@ -33,8 +33,14 @@ import ( "github.com/klauspost/compress/zstd" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" + atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" "go.ate.dev/substrate/cmd/ate-setup/internal/config" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/cmd/ate-setup/internal/log" @@ -450,8 +456,9 @@ func TestMicroVMSandboxConfigTemplate(t *testing.T) { if err != nil { t.Fatalf("DecodeManifestBytes() = %v", err) } - if len(objs) != 1 || objs[0].GetKind() != "SandboxConfig" || objs[0].GetName() != microvmSandboxConfigName { - t.Errorf("decoded objects = %v, want single SandboxConfig/%s", objs, microvmSandboxConfigName) + wantGVK := atev1.GroupVersion.WithKind("SandboxConfig") + if len(objs) != 1 || objs[0].GroupVersionKind() != wantGVK || objs[0].GetName() != microvmSandboxConfigName { + t.Errorf("decoded objects = %v, want single %v/%s", objs, wantGVK, microvmSandboxConfigName) } }) } @@ -543,6 +550,18 @@ func TestMicroVMBucket(t *testing.T) { wantBucket: "my-gke-bucket-assets", wantScheme: "gs", }, + { + name: "Non-Kind accepts 56-character BucketName (63-character asset bucket)", + cfg: config.Config{Kind: false, BucketName: strings.Repeat("a", 56)}, + wantBucket: strings.Repeat("a", 56) + "-assets", + wantScheme: "gs", + }, + { + name: "Non-Kind rejects BucketName whose asset bucket exceeds 63 characters", + cfg: config.Config{Kind: false, BucketName: strings.Repeat("a", 57)}, + wantScheme: "gs", + wantErr: true, + }, } { t.Run(tc.name, func(t *testing.T) { env := &Env{Cfg: &tc.cfg} @@ -560,6 +579,34 @@ func TestMicroVMBucket(t *testing.T) { } } +func TestDeleteMicroVMDeps(t *testing.T) { + silenceStepLog(t) + + gvr := atev1.GroupVersion.WithResource("sandboxconfigs") + cs := fake.NewClientset() + cs.Resources = []*metav1.APIResourceList{{ + GroupVersion: atev1.GroupVersion.String(), + APIResources: []metav1.APIResource{{Name: "sandboxconfigs", Kind: "SandboxConfig"}}, + }} + scheme := runtime.NewScheme() + dc := dynamicfake.NewSimpleDynamicClientWithCustomListKinds( + scheme, + map[schema.GroupVersionResource]string{gvr: "SandboxConfigList"}, + newSandboxConfig(microvmSandboxConfigName), + ) + env := &Env{ + Cfg: &config.Config{Kind: true}, + Kube: kube.NewFromClients(nil, cs, dc, cs.Discovery()), + } + + if err := env.DeleteMicroVMDeps(t.Context()); err != nil { + t.Fatalf("DeleteMicroVMDeps() = %v, want nil", err) + } + if _, err := dc.Resource(gvr).Get(t.Context(), microvmSandboxConfigName, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("Get(%s) after DeleteMicroVMDeps = %v, want NotFound", microvmSandboxConfigName, err) + } +} + func TestWaitJobComplete(t *testing.T) { silenceStepLog(t) From 5c731e88a343298453d6222ea9697e82bfe95220 Mon Sep 17 00:00:00 2001 From: Noah West Date: Fri, 23 Oct 2026 12:05:14 -0700 Subject: [PATCH 08/10] ate-setup: standardize added docs and comments on microVM AGENTS.md now standardizes prose on microVM without a hyphen. Update the docs, comments, and help text introduced by this branch to match. Signed-off-by: Noah West --- benchmarking/automation/README.md | 2 +- cmd/ate-setup/commands.md | 2 +- cmd/ate-setup/differences.md | 8 ++++---- cmd/ate-setup/internal/cmd/deploy.go | 6 +++--- .../internal/demos/countermicrovm/countermicrovm.go | 2 +- .../internal/demos/egressmicrovm/egressmicrovm.go | 2 +- cmd/ate-setup/internal/steps/deploy.go | 4 ++-- cmd/ate-setup/internal/steps/microvm.go | 12 ++++++------ hack/run-microvm-demo.sh | 4 ++-- .../ate-install/sandboxconfig-microvm.yaml.tmpl | 2 +- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index b84b9f662a..4a8f30fc94 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -21,7 +21,7 @@ the router capacity benchmark — see 5. `hack/install-ate.sh deploy ate-system` (with `ATE_CREDENTIAL_PROVIDER` in `.ate-dev-env.sh`, or `--credential-provider` in the test's `ateArgs`; passes `--skip-microvm-deps` unless `sandboxClass` is `microvm`, - in which case `deploy ate-system` also stages the micro-VM guest assets to + in which case `deploy ate-system` also stages the microVM guest assets to `-assets` and applies the cluster-wide `microvm` `SandboxConfig`) + `benchmarking/workloads/deploy.sh --deploy --sandbox-class ` (these build & push substrate / workload diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 1e6c6f4728..d7edd3f138 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -120,7 +120,7 @@ that already names a manifest is used as written, and is not looked up. `deploy ate-system` is the whole control plane: CRDs, RBAC, the store, the apiserver, the controller, atenet, and atelet. Unless `--skip-microvm-deps` is -passed (also accepted by `deploy sandboxconfig`), it also stages the micro-VM +passed (also accepted by `deploy sandboxconfig`), it also stages the microVM guest assets into the asset bucket (`rustfs` on Kind, `-assets` on GKE) and applies the `microvm` cluster-wide `SandboxConfig` alongside `gvisor-default`. It creates every `create` resource below on the way, so those diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 20cc17e695..ecaeddb805 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -4,7 +4,7 @@ `hack/install-demo-*.sh` scripts it sourced, `hack/setup-csi-*-kind.sh`, and `hack/install-microvm-deps.sh`. -The port has replaced them. The demo, CSI, and micro-VM asset scripts are +The port has replaced them. The demo, CSI, and microVM asset scripts are deleted, and `hack/install-ate.sh` forwards its arguments directly to `ate-setup`. See [`commands.md`](commands.md) for the historical flag mapping. @@ -84,8 +84,8 @@ Still required, and why: | `ko` | building and publishing images (`ko resolve` only); not used with `--image-repo` | | `go` | locating the pinned `ko` tool, exactly as `hack/run-tool.sh` does; not used with `--image-repo` | | `git` | the `git describe` version stamp passed to ko; not used with `--image-repo` | -| `docker` | the kind CSI setup (`docker exec` into the node), staging micro-VM assets into Kind's `rustfs`, and the claude-code-multiplex workload build | -| `gcloud` | GKE `get-credentials` (when `PROJECT_ID` is set and no context was given) and `gcloud storage` when staging micro-VM assets on GKE | +| `docker` | the kind CSI setup (`docker exec` into the node), staging microVM assets into Kind's `rustfs`, and the claude-code-multiplex workload build | +| `gcloud` | GKE `get-credentials` (when `PROJECT_ID` is set and no context was given) and `gcloud storage` when staging microVM assets on GKE | | `bash` | sourcing `.ate-dev-env.sh` | One shell script is still invoked rather than reimplemented, because it @@ -94,7 +94,7 @@ orchestrates benchmark image builds and Locust stack deployment: reconstructs the environment the shell installer would have exported to it, and is fronted by `deploy`/`delete benchmarks`. -The micro-VM asset assembly (`kata-static` `.tar.zst` extraction and +The microVM asset assembly (`kata-static` `.tar.zst` extraction and `cloud-hypervisor` download), object-store staging (`rustfs` on Kind, `-assets` on GKE), and `microvm` `SandboxConfig` lifecycle now run in Go as part of `deploy ate-system` (and `deploy sandboxconfig`), with diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index da01aff4ba..5980259d90 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -59,7 +59,7 @@ var deployAteSystemCmd = &cobra.Command{ This installs the CRDs and RBAC, the podcertificate controller and the secrets it signs, PostgreSQL, ate-api-server, ate-idp-server, ate-controller, the atenet -dataplane, the default gVisor and micro-VM SandboxConfigs (staging the micro-VM +dataplane, the default gVisor and microVM SandboxConfigs (staging the microVM guest assets to the cluster object store), and the atelet DaemonSet, then waits for each to roll out. @@ -146,7 +146,7 @@ var deployPodCertControllerCmd = &cobra.Command{ var deploySandboxConfigCmd = &cobra.Command{ Use: "sandboxconfig", - Short: "Deploy the SandboxConfig admission policy and the default gVisor and micro-VM SandboxConfigs only", + Short: "Deploy the SandboxConfig admission policy and the default gVisor and microVM SandboxConfigs only", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return env.DeploySandboxConfig(cmd.Context(), env.Cfg.Resolved().Bool("skipMicroVMDeps")) @@ -192,7 +192,7 @@ func init() { Key: "skipMicroVMDeps", Env: "ATE_INSTALL_SKIP_MICROVM_DEPS", Flag: "skip-microvm-deps", Kind: config.KindBool, Default: "false", Commands: []string{deployAteSystemPath, deploySandboxConfigPath}, - Usage: "Skip staging micro-VM guest assets and applying the microvm SandboxConfig", + Usage: "Skip staging microVM guest assets and applying the microvm SandboxConfig", }, ) config.BindCommandFlags(deployAteSystemPath, deployAteSystemCmd.Flags()) diff --git a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go index 812c6836fb..3a264c0165 100644 --- a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go +++ b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go @@ -30,7 +30,7 @@ const namespace = "ate-demo-counter-microvm" func init() { demos.Register(&demos.Substrate{ DemoName: "demo-counter-microvm", - Short: "The counter demo on micro-VM workers", + Short: "The counter demo on microVM workers", WorkerPoolManifest: "demos/counter/counter-microvm.yaml.tmpl", Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "counter-microvm"}}, Templates: []demos.SubstrateTemplate{{ diff --git a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go index 9047fdef98..d5b9276bba 100644 --- a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go +++ b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go @@ -33,7 +33,7 @@ const namespace = "ate-demo-egress-microvm" func init() { demos.Register(&demos.Substrate{ DemoName: "demo-egress-microvm", - Short: "Egress policy enforcement on micro-VM workers", + Short: "Egress policy enforcement on microVM workers", WorkerPoolManifest: "demos/egress/egress-microvm.yaml.tmpl", Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "egress-microvm"}}, Templates: []demos.SubstrateTemplate{{ diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index 5065378e8a..b341db86b3 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -43,7 +43,7 @@ type DeployOptions struct { // SetupCSI additionally installs the CSI driver (nfs, hostpath, both, none). // Kind only. The hostpath driver is Kind only. SetupCSI string - // SkipMicroVMDeps skips staging the micro-VM guest assets and applying the + // SkipMicroVMDeps skips staging the microVM guest assets and applying the // microvm SandboxConfig. SkipMicroVMDeps bool } @@ -594,7 +594,7 @@ func (e *Env) DeployAtenet(ctx context.Context) error { } // DeploySandboxConfig applies the SandboxConfig admission policy, then the -// default gVisor and (unless skipMicroVMDeps is set) micro-VM SandboxConfigs. +// default gVisor and (unless skipMicroVMDeps is set) microVM SandboxConfigs. func (e *Env) DeploySandboxConfig(ctx context.Context, skipMicroVMDeps bool) error { log.Step("deploy_sandboxconfig") diff --git a/cmd/ate-setup/internal/steps/microvm.go b/cmd/ate-setup/internal/steps/microvm.go index 0c506b1bc6..bbecd8888c 100644 --- a/cmd/ate-setup/internal/steps/microvm.go +++ b/cmd/ate-setup/internal/steps/microvm.go @@ -96,7 +96,7 @@ var ( // microvmReleaseBaseURL is overridden in unit tests to point at an httptest.Server. var microvmReleaseBaseURL = "https://github.com" -// DeployMicroVMDeps installs the micro-VM prerequisites: the guest assets for +// DeployMicroVMDeps installs the microVM prerequisites: the guest assets for // each supported architecture and the cluster-wide `microvm` SandboxConfig. func (e *Env) DeployMicroVMDeps(ctx context.Context) error { if err := ctx.Err(); err != nil { @@ -117,7 +117,7 @@ func (e *Env) DeployMicroVMDeps(ctx context.Context) error { log.Infof("Asset set in %s is stale (version stamp mismatch); re-assembling.", outDir) } if needAssemble { - log.Infof("Assembling micro-VM assets into %s (arch=%s)...", outDir, arch) + log.Infof("Assembling microVM assets into %s (arch=%s)...", outDir, arch) if err := assembleMicroVMAssets(ctx, outDir, arch, defaultKataVersion, defaultCloudHypervisorVersion); err != nil { return err } @@ -183,7 +183,7 @@ func (e *Env) DeleteMicroVMDeps(ctx context.Context) error { return nil } -// microVMBucket returns the bucket where micro-VM guest assets are staged: the +// microVMBucket returns the bucket where microVM guest assets are staged: the // shared rustfs bucket on Kind, and -assets on GKE (where atelet // has read access only to the asset bucket). func (e *Env) microVMBucket() (string, error) { @@ -194,7 +194,7 @@ func (e *Env) microVMBucket() (string, error) { return defaultKindMicroVMBucket, nil } if e.Cfg.BucketName == "" { - return "", fmt.Errorf("BUCKET_NAME (--storage-bucket-name or config.storage.bucketName) must be set to stage micro-VM assets on non-Kind clusters, or pass --skip-microvm-deps") + return "", fmt.Errorf("BUCKET_NAME (--storage-bucket-name or config.storage.bucketName) must be set to stage microVM assets on non-Kind clusters, or pass --skip-microvm-deps") } assetBucket := e.Cfg.BucketName + "-assets" if len(assetBucket) > maxGCSBucketNameLength { @@ -240,7 +240,7 @@ func checkMicroVMAssets(outDir string) error { func verifyMicroVMAssetDigests(outDir, arch string) error { digests, ok := microvmAssetDigests[arch] if !ok { - return fmt.Errorf("no pinned micro-VM asset digests for arch %q", arch) + return fmt.Errorf("no pinned microVM asset digests for arch %q", arch) } for _, f := range microvmAssets { p := filepath.Join(outDir, f) @@ -249,7 +249,7 @@ func verifyMicroVMAssetDigests(outDir, arch string) error { return err } if want := digests[f]; got != want { - return fmt.Errorf("micro-VM asset %s sha256 = %s, want %s (pinned in %s)", p, got, want, microvmSandboxConfigTemplate) + return fmt.Errorf("microVM asset %s sha256 = %s, want %s (pinned in %s)", p, got, want, microvmSandboxConfigTemplate) } } return nil diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 751018a337..373973a43f 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -19,7 +19,7 @@ # and calls this script), mirroring install-ate.sh / install-ate-kind.sh. # # Composes: -# 1. hack/install-ate.sh deploy ate-system (control plane + micro-VM assets and SandboxConfig) +# 1. hack/install-ate.sh deploy ate-system (control plane + microVM assets and SandboxConfig) # 2. hack/install-ate.sh deploy demo counter-microvm (worker pool manifest, atespace, and # ActorTemplate through the ate API) # @@ -74,7 +74,7 @@ log() { echo -e "${COLOR_CYAN}[run-microvm-demo]: $*${COLOR_RESET}" } -# --- 1. deploy the control plane (includes micro-VM assets + SandboxConfig) - +# --- 1. deploy the control plane (includes microVM assets + SandboxConfig) - log "Deploying the ate control plane (deploy ate-system)..." if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ATE_INSTALL_KIND itself. diff --git a/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl b/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl index b302788733..2134e7532d 100644 --- a/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Cluster-wide SandboxConfig for the micro-VM (kata + cloud-hypervisor) sandbox +# Cluster-wide SandboxConfig for the microVM (kata + cloud-hypervisor) sandbox # class. `go run ./cmd/ate-setup deploy ate-system` (and `deploy sandboxconfig`) # assembles the asset set, stages it to the cluster's object store, and applies # this manifest alongside sandboxconfig-gvisor.yaml. A microvm WorkerPool From c73202f1cd4a5256d3f55c6315b20b3d43ff1684 Mon Sep 17 00:00:00 2001 From: Noah West Date: Mon, 26 Oct 2026 08:51:42 -0700 Subject: [PATCH 09/10] ate-setup: update microVM assets to v2 and Kata 4.3.1 Carry over the Kata 4.3.1 v2 digests and amazon/aws-cli 2.38.5 image digest from the removed microvm-assets shell scripts. Signed-off-by: Noah West --- cmd/ate-setup/internal/steps/microvm.go | 16 ++++++++-------- cmd/ate-setup/internal/steps/microvm_test.go | 12 ++++++------ docs/dev/microvm-local.md | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cmd/ate-setup/internal/steps/microvm.go b/cmd/ate-setup/internal/steps/microvm.go index bbecd8888c..9fb0d29a2c 100644 --- a/cmd/ate-setup/internal/steps/microvm.go +++ b/cmd/ate-setup/internal/steps/microvm.go @@ -49,8 +49,8 @@ const ( // and bucket-init Job on Kind. MicroVMStageTimeout = 300 * time.Second - defaultMicroVMAssetVersion = "v1" - defaultKataVersion = "4.1.0" + defaultMicroVMAssetVersion = "v2" + defaultKataVersion = "4.3.1" defaultCloudHypervisorVersion = "v53.0" assetStampFile = ".asset-versions" @@ -61,7 +61,7 @@ const ( microvmSandboxConfigName = "microvm" // Keep in sync with manifests/ate-install/kind/rustfs.yaml. - rustfsAWSCLIImage = "amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73" + rustfsAWSCLIImage = "amazon/aws-cli:2.38.5@sha256:c8156a0e74e13929a42d7e60f0d4e14e7d28d980cd21822505d4bff80e5460c1" kataSharePrefix = "opt/kata/share/kata-containers/" kataKernelSymlink = "opt/kata/share/kata-containers/vmlinux.container" @@ -80,15 +80,15 @@ var ( microvmAssetDigests = map[string]map[string]string{ "arm64": { "cloud-hypervisor": "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850", - "virtiofsd": "3aa9240045d2261930ef04138fe05cf288f94c867e63e5d0845758bac5e3d458", - "vmlinux": "4960f6cf781274eaa445fc9f88bb8c5ce951649278bc9caa13afb1e99cfe80e6", - "rootfs.img": "7e3bdd500eb92de17c377bd84f8ed57e50d6bb68c7ec08c88f5e1916cd49f30d", + "virtiofsd": "4b43f4f6de6500203b01214a790314eea4182f5f110594ddb28ae2805622f06f", + "vmlinux": "dece388830720fe8bddb41552fff5f486240aeece5acc0aebc1812761e3e3f66", + "rootfs.img": "69f9a76e5a2563b7c74d1f271f3a904240f50ef405465f832021525894519f97", }, "amd64": { "cloud-hypervisor": "448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc", "virtiofsd": "15b2e72a78cc08a9bd8a6943e89fb69c88cb3cbeb63069efceade835342ac7d4", - "vmlinux": "8e9dbc3a6e4c26adb089d23d31201edebce905c7f3e31aa67b32bdd41df1b86f", - "rootfs.img": "96497f64da1de9c7473fef46c3d29ddd0805d334731cf9d903a21a5b2c33cefb", + "vmlinux": "f87d8359035cb7557b16a319e6431ea09b183e62268aef436bf8dc6530d786b6", + "rootfs.img": "6770db2cca5710593f27c9ac5e1d9d7aab836422695a5e5c4807496a92a0f24d", }, } ) diff --git a/cmd/ate-setup/internal/steps/microvm_test.go b/cmd/ate-setup/internal/steps/microvm_test.go index 9488ac21df..f559dd7778 100644 --- a/cmd/ate-setup/internal/steps/microvm_test.go +++ b/cmd/ate-setup/internal/steps/microvm_test.go @@ -122,7 +122,7 @@ func TestVerifyMicroVMAssetDigests(t *testing.T) { func TestMicroVMAssetsNeedAssemble(t *testing.T) { stamp := microvmAssetStamp("amd64", defaultKataVersion, defaultCloudHypervisorVersion) - const wantStamp = "arch=amd64\nkata=4.1.0\ncloud-hypervisor=v53.0\n" + const wantStamp = "arch=amd64\nkata=4.3.1\ncloud-hypervisor=v53.0\n" if stamp != wantStamp { t.Fatalf("microvmAssetStamp() =\n%q\nwant\n%q", stamp, wantStamp) } @@ -273,7 +273,7 @@ func serveFakeReleases(t *testing.T, archive []byte, chBody string, failCH bool) t.Helper() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { - case "/kata-containers/kata-containers/releases/download/4.1.0/kata-static-4.1.0-amd64.tar.zst": + case "/kata-containers/kata-containers/releases/download/4.3.1/kata-static-4.3.1-amd64.tar.zst": _, _ = w.Write(archive) case "/cloud-hypervisor/cloud-hypervisor/releases/download/v53.0/cloud-hypervisor-static": if failCH { @@ -308,7 +308,7 @@ func TestAssembleMicroVMAssets(t *testing.T) { serveFakeReleases(t, validKataArchive(t, virtiofsdBody), chBody, false) outDir := t.TempDir() - if err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.1.0", "v53.0"); err != nil { + if err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.3.1", "v53.0"); err != nil { t.Fatalf("assembleMicroVMAssets() = %v", err) } @@ -341,7 +341,7 @@ func TestAssembleMicroVMAssets(t *testing.T) { if err != nil { t.Fatalf("reading %s: %v", assetStampFile, err) } - if want := microvmAssetStamp("amd64", "4.1.0", "v53.0"); string(stamp) != want { + if want := microvmAssetStamp("amd64", "4.3.1", "v53.0"); string(stamp) != want { t.Errorf("stamp = %q, want %q", stamp, want) } } @@ -357,7 +357,7 @@ func TestAssembleMicroVMAssetsClearsStampOnFailure(t *testing.T) { t.Fatal(err) } - if err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.1.0", "v53.0"); err == nil { + if err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.3.1", "v53.0"); err == nil { t.Fatal("assembleMicroVMAssets() succeeded when cloud-hypervisor download returned 500") } if _, err := os.Stat(stampPath); !errors.Is(err, os.ErrNotExist) { @@ -371,7 +371,7 @@ func TestAssembleMicroVMAssetsRejectsDigestMismatchBeforeStamping(t *testing.T) serveFakeReleases(t, validKataArchive(t, "bad-virtiofsd-bytes"), "ch", false) outDir := t.TempDir() - err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.1.0", "v53.0") + err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.3.1", "v53.0") if err == nil || !strings.Contains(err.Error(), "sha256 =") { t.Fatalf("assembleMicroVMAssets() = %v, want sha256 mismatch error", err) } diff --git a/docs/dev/microvm-local.md b/docs/dev/microvm-local.md index d03512362e..ef2983a5cc 100644 --- a/docs/dev/microvm-local.md +++ b/docs/dev/microvm-local.md @@ -27,7 +27,7 @@ asset set is four files per architecture (`amd64` and `arm64`): - `rootfs.img` — the guest rootfs image (from `kata-static`) `go run ./cmd/ate-setup deploy ate-system` (and `deploy sandboxconfig`) -assembles both architectures into `bin/microvm-assets/v1/{amd64,arm64}/`, verifies +assembles both architectures into `bin/microvm-assets/v2/{amd64,arm64}/`, verifies their SHA-256 digests against `manifests/ate-install/sandboxconfig-microvm.yaml.tmpl`, stages them into the cluster's object store under `microvm-assets///` (the shared `rustfs` bucket on kind, and `-assets` on GKE, the only @@ -179,7 +179,7 @@ cd ``` The first run of `run-microvm-demo-kind.sh` downloads the guest runtime -releases and extracts them into `bin/microvm-assets/v1/{amd64,arm64}/`; later +releases and extracts them into `bin/microvm-assets/v2/{amd64,arm64}/`; later runs reuse those directories when the version stamps and SHA-256 digests match. Verify as in Option A, step 4. From 838f307d7cef721ba3cebee85d366342fa29b830 Mon Sep 17 00:00:00 2001 From: Noah West Date: Tue, 27 Oct 2026 10:43:29 -0700 Subject: [PATCH 10/10] ate-setup: retry transient Job waits and avoid duplicate CI microVM staging kube.Client.WaitJobComplete now polls through poll so transient apiserver errors retry while waiting for rustfs-bucket-init. pr-workflow.yaml runs deploy demo counter-microvm directly because deploy ate-system already deploys the control plane and stages microVM assets. assembleMicroVMAssets stages cloud-hypervisor in workDir before renaming into outDir so aborted downloads leave no partial binary behind. checkMicroVMDeps validates the asset bucket and staging CLI before DeployAteSystem, DeploySandboxConfig, or DeployMicroVMDeps mutates the cluster or downloads archives. stageMicroVMAssetsToRustfs creates custom buckets on Kind when BucketName differs from ate-snapshots, and TestDeploySkipMicroVMDepsFlag resets Cobra flags for shuffled test runs. Signed-off-by: Noah West --- .github/workflows/pr-workflow.yaml | 17 ++-- cmd/ate-setup/internal/cmd/deploy_test.go | 28 ++++--- cmd/ate-setup/internal/kube/wait.go | 38 +++++++++ cmd/ate-setup/internal/kube/wait_test.go | 75 ++++++++++++++++++ cmd/ate-setup/internal/steps/deploy.go | 11 +++ cmd/ate-setup/internal/steps/microvm.go | 67 ++++++++-------- cmd/ate-setup/internal/steps/microvm_test.go | 82 ++++++++++---------- 7 files changed, 225 insertions(+), 93 deletions(-) diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 70a43db96b..77205dbcf9 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -192,9 +192,8 @@ jobs: env: E2E_ATENET_DATAPLANE: ${{ matrix.dataplane }} # The installer requires a credential provider selection, and deploys the - # bundled Kubernetes Secrets provider with the control plane. Job-wide - # because every ate-setup invocation reads it, including the control - # plane redeploy in run-microvm-demo-kind.sh. + # bundled Kubernetes Secrets provider with the control plane. Job-wide so + # every ate-setup invocation in this job sees the same selection. ATE_CREDENTIAL_PROVIDER: '{"name":"k8s.io"}' # Job-wide for the same reason. Empty is an explicit "no preview # features", not an unset value. The e2e steps pass it to the suites as @@ -302,12 +301,14 @@ jobs: KUBECTL_CONTEXT: "kind-kind" run: go run ./cmd/ate-setup setup csi nfs - name: Deploy micro-VM counter demo - # Stages the (cached) assets into the cluster's rustfs and deploys the - # counter-microvm demo onto the control plane installed above. The demo - # redeploys the control plane, so retain the selected dataplane. env: - ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} - run: hack/run-microvm-demo-kind.sh + NO_DEV_ENV: "true" + KO_DOCKER_REPO: "localhost:5001" + ATE_INSTALL_KIND: "true" + BUCKET_NAME: "ate-snapshot" + KIND_CLUSTER_NAME: "kind" + KUBECTL_CONTEXT: "kind-kind" + run: go run ./cmd/ate-setup deploy demo counter-microvm - name: Deploy gVisor counter demo env: NO_DEV_ENV: "true" diff --git a/cmd/ate-setup/internal/cmd/deploy_test.go b/cmd/ate-setup/internal/cmd/deploy_test.go index 3fd579f36f..f4447db96e 100644 --- a/cmd/ate-setup/internal/cmd/deploy_test.go +++ b/cmd/ate-setup/internal/cmd/deploy_test.go @@ -17,6 +17,8 @@ package cmd import ( "testing" + "github.com/spf13/pflag" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) @@ -48,13 +50,21 @@ func TestComponentDeploySubcommands(t *testing.T) { } } -func TestDeploySkipMicroVMDepsFlag(t *testing.T) { - t.Run("deploy ate-system", func(t *testing.T) { - f := deployAteSystemCmd.Flags().Lookup("skip-microvm-deps") - t.Cleanup(func() { - _ = f.Value.Set("false") +func resetCommandFlags(t *testing.T, flags *pflag.FlagSet) { + t.Helper() + reset := func() { + flags.VisitAll(func(f *pflag.Flag) { + _ = f.Value.Set(f.DefValue) f.Changed = false }) + } + reset() + t.Cleanup(reset) +} + +func TestDeploySkipMicroVMDepsFlag(t *testing.T) { + t.Run("deploy ate-system", func(t *testing.T) { + resetCommandFlags(t, deployAteSystemCmd.Flags()) if err := deployAteSystemCmd.ParseFlags([]string{"--skip-microvm-deps"}); err != nil { t.Fatalf("ParseFlags(--skip-microvm-deps): %v", err) } @@ -79,14 +89,10 @@ func TestDeploySkipMicroVMDepsFlag(t *testing.T) { }) t.Run("deploy sandboxconfig", func(t *testing.T) { - f := deploySandboxConfigCmd.Flags().Lookup("skip-microvm-deps") - if f == nil { + if deploySandboxConfigCmd.Flags().Lookup("skip-microvm-deps") == nil { t.Fatal("deploy sandboxconfig is missing --skip-microvm-deps flag") } - t.Cleanup(func() { - _ = f.Value.Set("false") - f.Changed = false - }) + resetCommandFlags(t, deploySandboxConfigCmd.Flags()) if err := deploySandboxConfigCmd.ParseFlags([]string{"--skip-microvm-deps"}); err != nil { t.Fatalf("ParseFlags(--skip-microvm-deps): %v", err) } diff --git a/cmd/ate-setup/internal/kube/wait.go b/cmd/ate-setup/internal/kube/wait.go index d14b0b54cd..e3b31dd689 100644 --- a/cmd/ate-setup/internal/kube/wait.go +++ b/cmd/ate-setup/internal/kube/wait.go @@ -16,12 +16,14 @@ package kube import ( "context" + "errors" "fmt" "time" "go.ate.dev/substrate/cmd/ate-setup/internal/log" "go.ate.dev/substrate/internal/clustertrustbundle" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -183,6 +185,42 @@ func (c *Client) WaitNamespaceActive(ctx context.Context, name string, timeout t return nil } +// WaitJobComplete blocks until the named Job reports Complete=True, replacing +// `kubectl wait --for=condition=complete job/`. It fails immediately if +// the Job reports Failed=True. +func (c *Client) WaitJobComplete(ctx context.Context, namespace, name string, timeout time.Duration) error { + defer log.Elapsed(time.Now(), fmt.Sprintf("wait for job/%s", name)) + err := poll(ctx, timeout, func(ctx context.Context) (bool, error) { + job, err := c.Typed.BatchV1().Jobs(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + for _, cond := range job.Status.Conditions { + if cond.Type == batchv1.JobComplete && cond.Status == corev1.ConditionTrue { + return true, nil + } + if cond.Type == batchv1.JobFailed && cond.Status == corev1.ConditionTrue { + detail := cond.Message + if detail == "" { + detail = cond.Reason + } + if detail != "" { + return false, fmt.Errorf("job failed: %s", detail) + } + return false, errors.New("job failed") + } + } + return false, nil + }) + if err != nil { + return fmt.Errorf("waiting for job %s/%s to complete: %w", namespace, name, err) + } + return nil +} + // WaitCondition blocks until a custom resource reports the named status // condition as True, replacing `kubectl wait --for=condition=`. func (c *Client) WaitCondition(ctx context.Context, gvk schema.GroupVersionKind, namespace, name, condition string, timeout time.Duration) error { diff --git a/cmd/ate-setup/internal/kube/wait_test.go b/cmd/ate-setup/internal/kube/wait_test.go index 484e6ece5a..e85d4ea4ae 100644 --- a/cmd/ate-setup/internal/kube/wait_test.go +++ b/cmd/ate-setup/internal/kube/wait_test.go @@ -25,14 +25,18 @@ import ( "testing" "time" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" kubefake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" + k8stesting "k8s.io/client-go/testing" ) // shortPollInterval makes the retry path observable without a real sleep. @@ -184,3 +188,74 @@ func TestWaitClusterTrustBundlesVersions(t *testing.T) { type bundleRoundTripper func(*http.Request) (*http.Response, error) func (f bundleRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestWaitJobComplete(t *testing.T) { + shortPollInterval(t) + jobsResource := schema.GroupResource{Group: "batch", Resource: "jobs"} + + t.Run("retries transient error and NotFound until JobComplete", func(t *testing.T) { + kc := kubefake.NewSimpleClientset() + calls := 0 + kc.PrependReactor("get", "jobs", func(k8stesting.Action) (bool, runtime.Object, error) { + calls++ + switch calls { + case 1: + return true, nil, apierrors.NewServiceUnavailable("apiserver is starting") + case 2: + return true, nil, apierrors.NewNotFound(jobsResource, "rustfs-bucket-init") + default: + return true, &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ate-system", Name: "rustfs-bucket-init"}, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + {Type: batchv1.JobComplete, Status: corev1.ConditionTrue}, + }, + }, + }, nil + } + }) + c := &Client{Typed: kc} + if err := c.WaitJobComplete(t.Context(), "ate-system", "rustfs-bucket-init", time.Minute); err != nil { + t.Fatalf("WaitJobComplete() = %v, want nil", err) + } + if calls != 3 { + t.Errorf("Get called %d times, want 3", calls) + } + }) + + t.Run("aborts immediately when JobFailed is True", func(t *testing.T) { + kc := kubefake.NewSimpleClientset(&batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ate-system", Name: "rustfs-bucket-init"}, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + {Type: batchv1.JobFailed, Status: corev1.ConditionTrue, Message: "BackoffLimitExceeded"}, + }, + }, + }) + c := &Client{Typed: kc} + err := c.WaitJobComplete(t.Context(), "ate-system", "rustfs-bucket-init", time.Minute) + if err == nil || !strings.Contains(err.Error(), "BackoffLimitExceeded") { + t.Fatalf("WaitJobComplete() = %v, want BackoffLimitExceeded error", err) + } + if got := len(kc.Actions()); got != 1 { + t.Errorf("got %d API actions, want 1", got) + } + }) + + t.Run("aborts immediately on permanent API error", func(t *testing.T) { + kc := kubefake.NewSimpleClientset() + calls := 0 + kc.PrependReactor("get", "jobs", func(k8stesting.Action) (bool, runtime.Object, error) { + calls++ + return true, nil, apierrors.NewForbidden(jobsResource, "rustfs-bucket-init", errors.New("nope")) + }) + c := &Client{Typed: kc} + err := c.WaitJobComplete(t.Context(), "ate-system", "rustfs-bucket-init", time.Minute) + if !apierrors.IsForbidden(err) { + t.Fatalf("WaitJobComplete() = %v, want wrapped Forbidden error", err) + } + if calls != 1 { + t.Errorf("Get called %d times, want 1", calls) + } + }) +} diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index b341db86b3..b6ac889ed1 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -90,6 +90,11 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if _, err := clusterSnapshotLocation(e.Cfg.SnapshotLocation, e.Cfg.PreviewGates); err != nil { return err } + if !opts.SkipMicroVMDeps { + if _, err := e.checkMicroVMDeps(); err != nil { + return err + } + } if err := e.requireNoWorkerPods(ctx); err != nil { return err } @@ -598,6 +603,12 @@ func (e *Env) DeployAtenet(ctx context.Context) error { func (e *Env) DeploySandboxConfig(ctx context.Context, skipMicroVMDeps bool) error { log.Step("deploy_sandboxconfig") + if !skipMicroVMDeps { + if _, err := e.checkMicroVMDeps(); err != nil { + return err + } + } + if err := e.EnsureCRDs(ctx); err != nil { return err } diff --git a/cmd/ate-setup/internal/steps/microvm.go b/cmd/ate-setup/internal/steps/microvm.go index 9fb0d29a2c..3fc7dbdc65 100644 --- a/cmd/ate-setup/internal/steps/microvm.go +++ b/cmd/ate-setup/internal/steps/microvm.go @@ -31,12 +31,9 @@ import ( "time" "github.com/klauspost/compress/zstd" - batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/util/wait" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" @@ -104,7 +101,7 @@ func (e *Env) DeployMicroVMDeps(ctx context.Context) error { } log.Step("deploy_microvm_deps") - bucket, err := e.microVMBucket() + bucket, err := e.checkMicroVMDeps() if err != nil { return err } @@ -183,6 +180,25 @@ func (e *Env) DeleteMicroVMDeps(ctx context.Context) error { return nil } +// checkMicroVMDeps validates the asset bucket configuration and staging CLI +// tool before cluster mutation or archive downloads begin. +func (e *Env) checkMicroVMDeps() (string, error) { + bucket, err := e.microVMBucket() + if err != nil { + return "", err + } + if e.Cfg.Kind { + if _, err := exec.LookPath("docker"); err != nil { + return "", fmt.Errorf("'docker' is required to run the S3 client but was not found in PATH: %w", err) + } + } else { + if _, err := exec.LookPath("gcloud"); err != nil { + return "", fmt.Errorf("'gcloud' is required to upload microVM assets to GCS but was not found in PATH: %w", err) + } + } + return bucket, nil +} + // microVMBucket returns the bucket where microVM guest assets are staged: the // shared rustfs bucket on Kind, and -assets on GKE (where atelet // has read access only to the asset bucket). @@ -312,7 +328,11 @@ func assembleMicroVMAssets(ctx context.Context, outDir, arch, kataVer, chVer str chURL := fmt.Sprintf("%s/cloud-hypervisor/cloud-hypervisor/releases/download/%s/%s", baseURL, chVer, chAsset) log.Infof(">> Downloading cloud-hypervisor %s (%s)...", chVer, chAsset) - if err := downloadToFile(ctx, chURL, filepath.Join(outDir, "cloud-hypervisor"), 0o755); err != nil { + chTmp := filepath.Join(workDir, "cloud-hypervisor") + if err := downloadToFile(ctx, chURL, chTmp, 0o755); err != nil { + return err + } + if err := os.Rename(chTmp, filepath.Join(outDir, "cloud-hypervisor")); err != nil { return err } @@ -491,17 +511,13 @@ func downloadToFile(ctx context.Context, url, dstPath string, mode fs.FileMode) } func (e *Env) stageMicroVMAssetsToRustfs(ctx context.Context, bucket string) error { - if _, err := exec.LookPath("docker"); err != nil { - return fmt.Errorf("'docker' is required to run the S3 client but was not found in PATH: %w", err) - } - ns := e.Namespace() timeout := e.Cfg.WaitTimeout(MicroVMStageTimeout) log.Infof(">> Waiting for rustfs in namespace %s...", ns) if err := e.Kube.RolloutStatus(ctx, kube.KindDeployment, ns, "rustfs", timeout); err != nil { return err } - if err := e.waitJobComplete(ctx, ns, "rustfs-bucket-init", timeout); err != nil { + if err := e.Kube.WaitJobComplete(ctx, ns, "rustfs-bucket-init", timeout); err != nil { return err } @@ -532,6 +548,15 @@ func (e *Env) stageMicroVMAssetsToRustfs(ctx context.Context, bucket string) err return e.runCmd(ctx, stdin, "docker", dockerArgs...) } + // rustfs-bucket-init creates only defaultKindMicroVMBucket. + if bucket != defaultKindMicroVMBucket { + if err := awsCLI(nil, "s3api", "head-bucket", "--bucket", bucket); err != nil { + if err := awsCLI(nil, "s3api", "create-bucket", "--bucket", bucket); err != nil { + return fmt.Errorf("while creating bucket %s in rustfs: %w", bucket, err) + } + } + } + log.Infof(">> Uploading assets to s3://%s/%s/%s/ via %s (netns of %s)...", bucket, microvmAssetsPrefix, defaultMicroVMAssetVersion, endpoint, node) for _, arch := range microvmArches { outDir := e.Cfg.Path("bin", microvmAssetsPrefix, defaultMicroVMAssetVersion, arch) @@ -554,28 +579,6 @@ func (e *Env) stageMicroVMAssetsToRustfs(ctx context.Context, bucket string) err return awsCLI(nil, "s3", "ls", "--recursive", fmt.Sprintf("s3://%s/%s/", bucket, microvmAssetsPrefix)) } -func (e *Env) waitJobComplete(ctx context.Context, namespace, name string, timeout time.Duration) error { - defer log.Elapsed(time.Now(), fmt.Sprintf("wait for job/%s", name)) - return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { - job, err := e.Kube.Typed.BatchV1().Jobs(namespace).Get(ctx, name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return false, nil - } - if err != nil { - return false, err - } - for _, cond := range job.Status.Conditions { - if cond.Type == batchv1.JobComplete && cond.Status == corev1.ConditionTrue { - return true, nil - } - if cond.Type == batchv1.JobFailed && cond.Status == corev1.ConditionTrue { - return false, fmt.Errorf("job %s/%s failed: %s", namespace, name, cond.Message) - } - } - return false, nil - }) -} - func (e *Env) stageMicroVMAssetsToGCS(ctx context.Context, bucket string) error { var projectFlag []string if e.Cfg.ProjectID != "" { diff --git a/cmd/ate-setup/internal/steps/microvm_test.go b/cmd/ate-setup/internal/steps/microvm_test.go index f559dd7778..3cbfe50c9d 100644 --- a/cmd/ate-setup/internal/steps/microvm_test.go +++ b/cmd/ate-setup/internal/steps/microvm_test.go @@ -28,11 +28,8 @@ import ( "path/filepath" "strings" "testing" - "time" "github.com/klauspost/compress/zstd" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -277,8 +274,12 @@ func serveFakeReleases(t *testing.T, archive []byte, chBody string, failCH bool) _, _ = w.Write(archive) case "/cloud-hypervisor/cloud-hypervisor/releases/download/v53.0/cloud-hypervisor-static": if failCH { - http.Error(w, "boom", http.StatusInternalServerError) - return + w.Header().Set("Content-Length", "100") + _, _ = w.Write([]byte("partial")) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + panic(http.ErrAbortHandler) } _, _ = w.Write([]byte(chBody)) default: @@ -358,11 +359,14 @@ func TestAssembleMicroVMAssetsClearsStampOnFailure(t *testing.T) { } if err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.3.1", "v53.0"); err == nil { - t.Fatal("assembleMicroVMAssets() succeeded when cloud-hypervisor download returned 500") + t.Fatal("assembleMicroVMAssets() succeeded when cloud-hypervisor download aborted mid-stream") } if _, err := os.Stat(stampPath); !errors.Is(err, os.ErrNotExist) { t.Errorf("stamp file after failed assemble: err = %v, want ErrNotExist", err) } + if _, err := os.Stat(filepath.Join(outDir, "cloud-hypervisor")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("partial cloud-hypervisor after aborted download: err = %v, want ErrNotExist", err) + } } func TestAssembleMicroVMAssetsRejectsDigestMismatchBeforeStamping(t *testing.T) { @@ -511,11 +515,40 @@ func TestStageMicroVMAssetsToGCS(t *testing.T) { func TestDeployMicroVMDepsRequiresBucketOnNonKind(t *testing.T) { silenceStepLog(t) - env := &Env{Cfg: &config.Config{Root: t.TempDir(), Kind: false}} + cs := fake.NewClientset() + env := &Env{ + Cfg: &config.Config{Root: t.TempDir(), Kind: false}, + Kube: kube.NewFromClients(nil, cs, nil, cs.Discovery()), + } err := env.DeployMicroVMDeps(t.Context()) if err == nil || !strings.Contains(err.Error(), "BUCKET_NAME") { t.Fatalf("DeployMicroVMDeps() on non-Kind without BucketName = %v, want BUCKET_NAME error", err) } + err = env.DeploySandboxConfig(t.Context(), false) + if err == nil || !strings.Contains(err.Error(), "BUCKET_NAME") { + t.Fatalf("DeploySandboxConfig() on non-Kind without BucketName = %v, want BUCKET_NAME error", err) + } + if got := len(cs.Actions()); got != 0 { + t.Errorf("DeploySandboxConfig() made %d API calls before preflight failed, want 0", got) + } +} + +func TestCheckMicroVMDepsRequiresStageTool(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + + t.Run("Kind requires docker", func(t *testing.T) { + env := &Env{Cfg: &config.Config{Kind: true}} + if _, err := env.checkMicroVMDeps(); err == nil || !strings.Contains(err.Error(), "docker") { + t.Fatalf("checkMicroVMDeps() on Kind without docker = %v, want docker error", err) + } + }) + + t.Run("Non-Kind requires gcloud", func(t *testing.T) { + env := &Env{Cfg: &config.Config{Kind: false, BucketName: "test-bucket"}} + if _, err := env.checkMicroVMDeps(); err == nil || !strings.Contains(err.Error(), "gcloud") { + t.Fatalf("checkMicroVMDeps() on non-Kind without gcloud = %v, want gcloud error", err) + } + }) } func TestMicroVMBucket(t *testing.T) { @@ -607,41 +640,6 @@ func TestDeleteMicroVMDeps(t *testing.T) { } } -func TestWaitJobComplete(t *testing.T) { - silenceStepLog(t) - - t.Run("returns nil when JobComplete is True", func(t *testing.T) { - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: "rustfs-bucket-init"}, - Status: batchv1.JobStatus{ - Conditions: []batchv1.JobCondition{ - {Type: batchv1.JobComplete, Status: corev1.ConditionTrue}, - }, - }, - } - e := &Env{Kube: fakeKube(t, job)} - if err := e.waitJobComplete(t.Context(), NamespaceAteSystem, "rustfs-bucket-init", time.Second); err != nil { - t.Errorf("waitJobComplete() = %v, want nil", err) - } - }) - - t.Run("returns error immediately when JobFailed is True", func(t *testing.T) { - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: "rustfs-bucket-init"}, - Status: batchv1.JobStatus{ - Conditions: []batchv1.JobCondition{ - {Type: batchv1.JobFailed, Status: corev1.ConditionTrue, Message: "BackoffLimitExceeded"}, - }, - }, - } - e := &Env{Kube: fakeKube(t, job)} - err := e.waitJobComplete(t.Context(), NamespaceAteSystem, "rustfs-bucket-init", time.Second) - if err == nil || !strings.Contains(err.Error(), "BackoffLimitExceeded") { - t.Errorf("waitJobComplete() = %v, want BackoffLimitExceeded error", err) - } - }) -} - func TestMicroVMDepsStepsAreSeparableFromContext(t *testing.T) { silenceStepLog(t)