diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 4584f1057b..b51e902ac9 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -192,9 +192,8 @@ jobs: env: E2E_ATENET_DATAPLANE: ${{ matrix.dataplane }} # The installer requires a credential provider selection, and deploys the - # bundled Kubernetes Secrets provider with the control plane. Job-wide - # because every ate-setup invocation reads it, including the control - # plane redeploy in run-microvm-demo-kind.sh. + # bundled Kubernetes Secrets provider with the control plane. Job-wide so + # every ate-setup invocation in this job sees the same selection. ATE_CREDENTIAL_PROVIDER: '{"name":"k8s.io"}' # Job-wide for the same reason. Empty is an explicit "no preview # features", not an unset value. The e2e steps pass it to the suites as @@ -258,13 +257,14 @@ jobs: id: microvm-assets uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - # Assembling the assets (download kata-static + cloud-hypervisor + the - # prebuilt virtiofsd — all downloads on amd64) is fully pinned by assemble.sh, - # so key the cache on its hash. The push-to-main run populates the + # Assembling the assets (download kata-static + cloud-hypervisor) is + # pinned by cmd/ate-setup/internal/steps/microvm.go and + # manifests/ate-install/sandboxconfig-microvm.yaml.tmpl, so key the + # cache on their hashes. The push-to-main run populates the # default-branch cache PRs inherit; the weekly schedule refreshes it. - # run-microvm-demo-kind.sh skips assembling when these are present. - path: bin/microvm-assets/amd64 - key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }} + # ate-setup skips assembling when these are present. + path: bin/microvm-assets + key: microvm-assets-${{ hashFiles('cmd/ate-setup/internal/steps/microvm.go', 'manifests/ate-install/sandboxconfig-microvm.yaml.tmpl') }} - name: Enable KVM # Grant the runner access to /dev/kvm so create-kind-cluster.sh mounts it into # the node and labels it for the micro-VM sandbox class. @@ -302,12 +302,14 @@ jobs: KUBECTL_CONTEXT: "kind-kind" run: go run ./cmd/ate-setup setup csi nfs - name: Deploy micro-VM counter demo - # Stages the (cached) assets into the cluster's rustfs and deploys the - # counter-microvm demo onto the control plane installed above. The demo - # redeploys the control plane, so retain the selected dataplane. env: - ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} - run: hack/run-microvm-demo-kind.sh + NO_DEV_ENV: "true" + KO_DOCKER_REPO: "localhost:5001" + ATE_INSTALL_KIND: "true" + BUCKET_NAME: "ate-snapshot" + KIND_CLUSTER_NAME: "kind" + KUBECTL_CONTEXT: "kind-kind" + run: go run ./cmd/ate-setup deploy demo counter-microvm - name: Deploy gVisor counter demo env: NO_DEV_ENV: "true" diff --git a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go index 6b6663cec9..a05a783020 100644 --- a/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go +++ b/api/infra/k8s/ate/v1/sandboxconfig_validation_test.go @@ -352,7 +352,7 @@ func TestShippedSandboxConfigManifests(t *testing.T) { for _, path := range []string{ "../../../../../manifests/ate-install/sandboxconfig-gvisor.yaml", - "../../../../../manifests/microvm/sandboxconfig-microvm.yaml.tmpl", + "../../../../../manifests/ate-install/sandboxconfig-microvm.yaml.tmpl", } { t.Run(path, func(t *testing.T) { raw, err := os.ReadFile(path) diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 7f36ddf5be..f93a0f957b 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -20,17 +20,17 @@ the router capacity benchmark — see and/or `${KO_DOCKER_REPO}/nighthawk-ingress-test:`. 5. `hack/install-ate.sh deploy ate-system` (with `ATE_CREDENTIAL_PROVIDER` in `.ate-dev-env.sh`, or `--credential-provider` in the test's - `ateArgs`) + `benchmarking/workloads/deploy.sh + `ateArgs`; passes `--sandbox-classes=` so `deploy ate-system` + installs the test's class and, for `microvm`, stages the microVM guest + assets to `-assets` and applies the cluster-wide `microvm` + `SandboxConfig`) + `benchmarking/workloads/deploy.sh --deploy --sandbox-class ` (these build & push substrate / workload images via `ko` as part of their deploy steps — there's no separate image build step). The install sets `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` to the `ate-client` and `benchmarking/benchmark-runner` ServiceAccounts so the runner's calls pass [authorization](../../docs/authorization.md), and sets `ATE_INGRESS_ALLOWED_SPIFFE_IDS` to `benchmarking/benchmark-runner` for the - Envoy ingress router. For a `microvm` test the orchestrator also - runs `hack/install-microvm-deps.sh --install` between the two, which - stages kata + cloud-hypervisor + virtiofsd assets to the cluster's object - store bucket and applies the cluster-wide `microvm` SandboxConfig. + Envoy ingress router. For a `nighthawk-ingress` test the orchestrator additionally patches the `atenet-router` Deployment right after `deploy_substrate`: `envoyCpu` is the benchmark's independent variable and the shipped manifest sets @@ -47,7 +47,7 @@ the router capacity benchmark — see variable (a JSON object) if set. A `--metadata` in the test's own `flags` is merged in too, its keys winning on conflict. - Polls until complete/failed/timeout; tails logs; deletes the Job. - - Tears down workloads + microVM deps (if any) + substrate. + - Tears down workloads + substrate. - If not the last test, redeploys them so the next run starts clean. ## Choosing a sandbox class diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index f60e90e455..b14f3c9316 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -391,10 +391,15 @@ def validate_and_normalize_tests(tests: list[dict[str, Any]]) -> None: def substrate_args(test: dict[str, Any]) -> list[str]: """Return the hack/install-ate.sh deploy ate-system arguments for a - tests.yaml entry: its ateArgs, then the fake data plane flag its isolate - block asks for. The teardown needs no matching flag: delete ate-system - removes the fake data plane whether or not it was installed.""" - return [*test.get("ateArgs", []), *isolate_install_args(test)] + tests.yaml entry: its ateArgs, the fake data plane flag its isolate block + asks for, and --sandbox-classes= so gVisor runs do not need + write access to -assets. The teardown needs no matching flag: + delete ate-system removes the fake data plane and any SandboxConfigs + whether or not they were installed.""" + args = [*test.get("ateArgs", []), *isolate_install_args(test)] + if not any(a == "--sandbox-classes" or a.startswith("--sandbox-classes=") for a in args): + args.append(f"--sandbox-classes={test.get('sandboxClass', 'gvisor')}") + return args # ate-api-server enforces authorization, and the runner Job calls it with its @@ -420,20 +425,6 @@ def teardown_substrate() -> None: run_no_check(["hack/install-ate.sh", "delete", "ate-system"]) -def install_microvm_deps() -> None: - """Stage kata/cloud-hypervisor assets and apply the cluster-wide - microvm SandboxConfig. Required before a microvm WorkerPool can run - actors; must run after deploy_substrate() (which installs the CRDs).""" - run(["hack/install-microvm-deps.sh", "--install"]) - - -def teardown_microvm_deps() -> None: - """Remove the microvm SandboxConfig. Must run before - teardown_substrate(), which deletes the SandboxConfig CRD (and would - prevent this from succeeding via kubectl).""" - run_no_check(["hack/install-microvm-deps.sh", "--delete"]) - - def deploy_workloads( worker_count: int = 1, sandbox_class: str = "gvisor", @@ -632,11 +623,8 @@ def main() -> None: # fire that crashed mid-test (or any other process that left # state behind) would otherwise leak its substrate + workloads # into this run. All teardowns use --ignore-not-found, so - # this is cheap on a clean cluster. Order matters: - # microvm-deps deletes a SandboxConfig CR, which requires the - # SandboxConfig CRD that teardown_substrate removes. + # this is cheap on a clean cluster. teardown_workloads() - teardown_microvm_deps() teardown_substrate() sandbox_class = test.get("sandboxClass", "gvisor") @@ -646,10 +634,6 @@ def main() -> None: try: deploy_substrate(substrate_args(test)) TYPES[ttype].pre_test(test) - # install-microvm-deps needs the CRDs from deploy_substrate; - # deploy_workloads needs the microvm SandboxConfig. - if sandbox_class == "microvm": - install_microvm_deps() deploy_workloads( test.get("workerCount", 1), sandbox_class, @@ -678,10 +662,7 @@ def main() -> None: finally: # Always tear down, even if deploy or run failed, so the # next test (and the next CronJob fire) starts clean. - # microvm-deps must go before substrate for the same reason - # as above. teardown_workloads() - teardown_microvm_deps() teardown_substrate() duration = time.time() - start_time results.append((test["name"], status, duration, failure_msg)) diff --git a/benchmarking/automation/test_orchestrator.py b/benchmarking/automation/test_orchestrator.py index 77833d43ba..c562fc8a05 100644 --- a/benchmarking/automation/test_orchestrator.py +++ b/benchmarking/automation/test_orchestrator.py @@ -40,7 +40,7 @@ class SubstrateArgsTest(unittest.TestCase): def test_ate_args_only(self): self.assertEqual( orchestrator.substrate_args({"ateArgs": ["--cluster-size=size10"]}), - ["--cluster-size=size10"], + ["--cluster-size=size10", "--sandbox-classes=gvisor"], ) def test_isolate_installs_the_fake_data_plane(self): @@ -50,14 +50,26 @@ def test_isolate_installs_the_fake_data_plane(self): "isolate": {"delay": "500ms"}, }), # The delay tunes fake-atelet through deploy.sh, not the install. - ["--cluster-size=size10", "--fake-data-plane"], + ["--cluster-size=size10", "--fake-data-plane", "--sandbox-classes=gvisor"], ) def test_neither(self): - self.assertEqual(orchestrator.substrate_args({}), []) + self.assertEqual(orchestrator.substrate_args({}), ["--sandbox-classes=gvisor"]) def test_bare_isolate_key(self): - self.assertEqual(orchestrator.substrate_args({"isolate": None}), ["--fake-data-plane"]) + self.assertEqual( + orchestrator.substrate_args({"isolate": None}), + ["--fake-data-plane", "--sandbox-classes=gvisor"], + ) + + def test_microvm_selects_microvm_sandbox_class(self): + self.assertEqual( + orchestrator.substrate_args({ + "sandboxClass": "microvm", + "ateArgs": ["--cluster-size=size10"], + }), + ["--cluster-size=size10", "--sandbox-classes=microvm"], + ) class DeploySubstrateTest(unittest.TestCase): diff --git a/benchmarking/deploy_locust.sh b/benchmarking/deploy_locust.sh index 6d216125cd..1dfdc96567 100755 --- a/benchmarking/deploy_locust.sh +++ b/benchmarking/deploy_locust.sh @@ -45,8 +45,7 @@ usage() { echo " --deploy Deploy workloads, build/push locust image, then deploy locust" echo " --delete Delete locust and then workloads" echo " --worker-count N Number of WorkerPool replicas (default: 1)" - echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)." - echo " microvm requires hack/install-microvm-deps.sh --install to have run." + echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)" echo " --otlp-endpoint URL Forwarded to workloads/deploy.sh. The address to which an" echo " instrumented actor container sends telemetry." echo " --actor-memory SIZE Forwarded to workloads/deploy.sh. Memory limit for the" diff --git a/benchmarking/workloads/deploy.sh b/benchmarking/workloads/deploy.sh index 010a2cc295..fccb9bc56e 100755 --- a/benchmarking/workloads/deploy.sh +++ b/benchmarking/workloads/deploy.sh @@ -93,8 +93,7 @@ usage() { echo " --deploy Substitute env vars and deploy workloads to the cluster using ko apply" echo " --delete Substitute env vars and delete workloads from the cluster" echo " --worker-count N Number of WorkerPool replicas (default: 1)" - echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)." - echo " microvm requires hack/install-microvm-deps.sh --install to have run." + echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)" echo " --actor-memory SIZE Memory limit for the benchmark ActorTemplates (default: 256Mi)" echo " --actor-cpu CPU CPU limit for the benchmark ActorTemplates (default: 1)" echo " --worker-memory SIZE Memory request and limit for each WorkerPool pod" diff --git a/cmd/ate-setup/README.md b/cmd/ate-setup/README.md index 17e1d8c248..2268903ae3 100644 --- a/cmd/ate-setup/README.md +++ b/cmd/ate-setup/README.md @@ -20,12 +20,15 @@ tag they share: ``` ate-setup deploy ate-system \ + --storage-bucket-name=BUCKET \ --credential-provider='{"name":"k8s.io"}' \ --ingress-allowed-spiffe-ids=IDS \ --image-repo registry.example.com/substrate \ --image-tag vX.Y.Z ``` +Pass `--sandbox-classes=gvisor` to skip staging microVM guest assets to `-assets`. + The registry has to hold every image the install deploys. The images a release publishes to `ghcr.io/agent-substrate/substrate` leave out the demos and the benchmarking fakes; see diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 267b790648..767dc8ac2d 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -68,6 +68,7 @@ unchanged. ``` ate-setup deploy ate-system \ + --storage-bucket-name=BUCKET \ --credential-provider='{"name":"k8s.io"}' \ --ingress-allowed-spiffe-ids=IDS \ --image-repo REPO \ @@ -77,7 +78,11 @@ ate-setup deploy ate-system \ installs images already published to `REPO` instead, and never invokes `ko`. The manifests still come from the checkout, so this needs one; what it removes is the build and write access to a registry. Releases ship no `ate-setup` -binary, so running it from the checkout still needs Go. +binary, so running it from the checkout still needs Go. On non-Kind clusters, +the default `--sandbox-classes=gvisor,microvm` also downloads the microVM guest +assets from GitHub and uploads them to `-assets` with `gcloud`; pass +`--sandbox-classes=gvisor` to install gVisor only without `--storage-bucket-name` +or `gcloud`. `REPO` has to hold every image the install deploys, all under `TAG`, including `envoy-dataplane`, which a build from source builds from a Dockerfile with @@ -109,18 +114,23 @@ that already names a manifest is used as written, and is not looked up. |---|---| | `deploy ate-system` | `--deploy-ate-system` | | `deploy ate-system --setup-csi=nfs` | `--deploy-ate-system --setup-csi=nfs` | +| `deploy ate-system --sandbox-classes=gvisor` | (no shell equivalent) | | `deploy atelet` | `--deploy-atelet` | | `deploy apiserver` | `--deploy-ate-apiserver` | | `deploy ate-controller` | (no shell equivalent) | | `deploy atenet` | `--deploy-atenet` | | `deploy podcertificate-controller` | (no shell equivalent) | -| `deploy sandboxconfig` | (no shell equivalent) | +| `deploy sandboxconfig [--sandbox-classes=...]` | (no shell equivalent) | | `deploy postgres` | (no shell equivalent) | `deploy ate-system` is the whole control plane: CRDs, RBAC, the store, the -apiserver, the controller, atenet, and atelet. It creates every `create` -resource below on the way, so those subcommands are only needed to redo one on -a running cluster. +apiserver, the controller, atenet, and atelet. By default +(`--sandbox-classes=gvisor,microvm`, also accepted by `deploy sandboxconfig`), +it installs the cluster-wide `gvisor-default` and `microvm` `SandboxConfig` +resources, staging the microVM guest assets into the asset bucket (`rustfs` on +Kind, `-assets` on GKE). It creates every `create` resource below +on the way, so those subcommands are only needed to redo one on a running +cluster. ## Publish @@ -210,9 +220,9 @@ See |---|---|---| | `deploy demo counter` | `--deploy-demo-counter` | A counter actor exercising snapshot, resume, and atenet ingress | | `deploy demo counter --with-external-volume [--storage-class NAME]` | `--deploy-demo-counter-with-external-volume` (`STORAGE_CLASS=NAME`) | The same, plus an external volume and a pre-seeded file to validate (requires `--preview=ExternalVolumes`). Run `setup csi` first and name the class it created, e.g. `csi-nfs-sc`; defaults to `standard` | -| `deploy demo counter-microvm` | `--deploy-demo-counter-microvm` | The counter demo on microVM workers. Run `hack/install-microvm-deps.sh --install` first | +| `deploy demo counter-microvm` | `--deploy-demo-counter-microvm` | The counter demo on microVM workers | | `deploy demo egress` | `--deploy-demo-egress` | Egress policy enforcement through atenet; the actors trust the egress gateway's CA | -| `deploy demo egress-microvm` | `--deploy-demo-egress-microvm` | The same on microVM workers. Run `hack/install-microvm-deps.sh --install` first | +| `deploy demo egress-microvm` | `--deploy-demo-egress-microvm` | The same on microVM workers | | `deploy demo jupyter` | `--deploy-demo-jupyter` | A Jupyter notebook server per actor, reached through atenet ingress | | `deploy demo sandbox` | `--deploy-demo-sandbox` | An on-demand sandbox actor driven by the sandbox client | | `deploy demo multi-template` | `--deploy-demo-multi-template` | Two ActorTemplates sharing one WorkerPool | diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 725594261f..ecaeddb805 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -1,13 +1,12 @@ # ate-setup vs. the install shell scripts `ate-setup` is a Go port of `hack/install-ate.sh`, the seven -`hack/install-demo-*.sh` scripts it sourced, and `hack/setup-csi-*-kind.sh`. +`hack/install-demo-*.sh` scripts it sourced, `hack/setup-csi-*-kind.sh`, and +`hack/install-microvm-deps.sh`. -The port has replaced them. The demo scripts are deleted, and -`hack/install-ate.sh` is a shim holding no install logic of its own: it -translates the flags and environment variables the installer has always -accepted onto `ate-setup` commands, so existing command lines keep working. See -[`commands.md`](commands.md) for the flag-by-flag mapping. +The port has replaced them. The demo, CSI, and microVM asset scripts are +deleted, and `hack/install-ate.sh` forwards its arguments directly to +`ate-setup`. See [`commands.md`](commands.md) for the historical flag mapping. This document covers what changed *behind* that mapping: the design decisions of the port, against the shell it replaced. For what a user of @@ -85,15 +84,22 @@ Still required, and why: | `ko` | building and publishing images (`ko resolve` only); not used with `--image-repo` | | `go` | locating the pinned `ko` tool, exactly as `hack/run-tool.sh` does; not used with `--image-repo` | | `git` | the `git describe` version stamp passed to ko; not used with `--image-repo` | -| `docker` | the kind CSI setup (`docker exec` into the node) and the claude-code-multiplex workload build | -| `gcloud` | GKE `get-credentials`, only when `PROJECT_ID` is set and no context was given | +| `docker` | the kind CSI setup (`docker exec` into the node), staging microVM assets into Kind's `rustfs`, and the claude-code-multiplex workload build | +| `gcloud` | GKE `get-credentials` (when `PROJECT_ID` is set and no context was given) and `gcloud storage` when staging microVM assets on GKE | | `bash` | sourcing `.ate-dev-env.sh` | -Two shell scripts are still invoked rather than reimplemented, because they -orchestrate image builds, asset assembly, and object-store staging that are out -of scope for an installer: `benchmarking/deploy_locust.sh` and -`hack/install-microvm-deps.sh`. They receive `Config.ScriptEnv()`, which -reconstructs the environment the shell installer would have exported to them. +One shell script is still invoked rather than reimplemented, because it +orchestrates benchmark image builds and Locust stack deployment: +`benchmarking/deploy_locust.sh`. It receives `Config.ScriptEnv()`, which +reconstructs the environment the shell installer would have exported to it, and +is fronted by `deploy`/`delete benchmarks`. + +The microVM asset assembly (`kata-static` `.tar.zst` extraction and +`cloud-hypervisor` download), object-store staging (`rustfs` on Kind, +`-assets` on GKE), and `microvm` `SandboxConfig` lifecycle now run +in Go as part of `deploy ate-system` (and `deploy sandboxconfig`), with +`delete ate-system` removing the `microvm` `SandboxConfig` alongside the rest +of the control plane. `ko` is no longer asked to apply anything. The scripts ran `run_ko apply`, which made ko shell out to kubectl and forced the awkward `-- --context=` special case diff --git a/cmd/ate-setup/internal/cmd/benchmarks.go b/cmd/ate-setup/internal/cmd/benchmarks.go index d92d1c4b6e..670cf9b495 100644 --- a/cmd/ate-setup/internal/cmd/benchmarks.go +++ b/cmd/ate-setup/internal/cmd/benchmarks.go @@ -82,11 +82,7 @@ See benchmarking/README.md for the walkthrough and customization options.`, var deleteBenchmarksCmd = &cobra.Command{ Use: "benchmarks", Short: "Delete the locust load test stack and the benchmark workloads", - Long: `Delete the locust load test stack and the benchmark workloads. - -Pass --sandbox-class=microvm to also remove the micro-VM SandboxConfig; it is -cluster-wide, so it is left in place by default.`, - Args: cobra.NoArgs, + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { opts, err := benchmarkOptions() if err != nil { diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 76d3f12ff0..92e64751e3 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -27,8 +27,11 @@ var deployCmd = &cobra.Command{ Short: "Deploy Agent Substrate components", } -// deployAteSystemPath is the command path the ate-system settings bind to. -const deployAteSystemPath = "deploy ate-system" +// Command paths the deploy subcommands bind their flags to. +const ( + deployAteSystemPath = "deploy ate-system" + deploySandboxConfigPath = "deploy sandboxconfig" +) // deployOptions reads the settings deploy ate-system owns. // @@ -38,14 +41,42 @@ const deployAteSystemPath = "deploy ate-system" // correct for an argument: there is no channel to name. func deployOptions() (steps.DeployOptions, error) { r := env.Cfg.Resolved() - opts := steps.DeployOptions{SetupCSI: r.String("csi.setup")} + opts := steps.DeployOptions{ + SetupCSI: r.String("csi.setup"), + } if err := steps.ValidateCSIDriver(opts.SetupCSI); err != nil { v, _ := r.Value("csi.setup") return opts, &config.InvalidError{Value: v, Want: "nfs, hostpath, both or none"} } + classes, err := sandboxClassesOption() + if err != nil { + return opts, err + } + opts.SandboxClasses = classes return opts, nil } +func sandboxClassesOption() (string, error) { + r := env.Cfg.Resolved() + classes := r.String("sandboxClasses") + if err := steps.ValidateSandboxClasses(classes); err != nil { + v, _ := r.Value("sandboxClasses") + return "", &config.InvalidError{Value: v, Want: "gvisor, microvm or gvisor,microvm"} + } + return classes, nil +} + +func validateSandboxClassesFlag(cmd *cobra.Command) error { + if !cmd.Flags().Changed("sandbox-classes") { + return nil + } + classes, err := cmd.Flags().GetString("sandbox-classes") + if err != nil { + return err + } + return steps.ValidateSandboxClasses(classes) +} + var deployAteSystemCmd = &cobra.Command{ Use: "ate-system", Short: "Deploy the core system: CRDs, RBAC, store, apiserver, controller, atenet, and atelet", @@ -53,7 +84,9 @@ var deployAteSystemCmd = &cobra.Command{ This installs the CRDs and RBAC, the podcertificate controller and the secrets it signs, PostgreSQL, ate-api-server, ate-idp-server, ate-controller, the atenet -dataplane, and the atelet DaemonSet, then waits for each to roll out. +dataplane, the default gVisor and microVM SandboxConfigs (staging the microVM +guest assets to the cluster object store), and the atelet DaemonSet, then waits +for each to roll out. The bundled PostgreSQL StatefulSet is skipped when ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING or the ATE_API_POSTGRES_CLOUDSQL_* variables @@ -70,6 +103,9 @@ Shape the install with the global --atenet-dataplane, --cluster-size, if err := cobra.NoArgs(cmd, args); err != nil { return err } + if err := validateSandboxClassesFlag(cmd); err != nil { + return err + } if !cmd.Flags().Changed("setup-csi") { return nil } @@ -138,10 +174,19 @@ var deployPodCertControllerCmd = &cobra.Command{ var deploySandboxConfigCmd = &cobra.Command{ Use: "sandboxconfig", - Short: "Deploy the SandboxConfig admission policy and the default gVisor SandboxConfig only", - Args: cobra.NoArgs, + Short: "Deploy the SandboxConfig admission policy and the default gVisor and microVM SandboxConfigs only", + Args: func(cmd *cobra.Command, args []string) error { + if err := cobra.NoArgs(cmd, args); err != nil { + return err + } + return validateSandboxClassesFlag(cmd) + }, RunE: func(cmd *cobra.Command, _ []string) error { - return env.DeploySandboxConfig(cmd.Context()) + classes, err := sandboxClassesOption() + if err != nil { + return err + } + return env.DeploySandboxConfig(cmd.Context(), classes) }, } @@ -173,12 +218,21 @@ func init() { deployPostgresCmd, ) - config.RegisterCommand(config.Setting{ - Key: "csi.setup", Env: "SETUP_CSI", Flag: "setup-csi", - Kind: config.KindString, Default: "none", - Commands: []string{deployAteSystemPath}, - Usage: "Also install CSI driver (nfs, hostpath, both, none; default: none)", - }) + config.RegisterCommand( + config.Setting{ + Key: "csi.setup", Env: "SETUP_CSI", Flag: "setup-csi", + Kind: config.KindString, Default: "none", + Commands: []string{deployAteSystemPath}, + Usage: "Also install CSI driver (nfs, hostpath, both, none; default: none)", + }, + config.Setting{ + Key: "sandboxClasses", Env: "ATE_INSTALL_SANDBOX_CLASSES", Flag: "sandbox-classes", + Kind: config.KindString, Default: steps.DefaultSandboxClasses, + Commands: []string{deployAteSystemPath, deploySandboxConfigPath}, + Usage: "Default SandboxConfigs to install (comma-separated: gvisor, microvm; default: gvisor,microvm)", + }, + ) config.BindCommandFlags(deployAteSystemPath, deployAteSystemCmd.Flags()) + config.BindCommandFlags(deploySandboxConfigPath, deploySandboxConfigCmd.Flags()) deployAteSystemCmd.Flags().Lookup("setup-csi").NoOptDefVal = "none" } diff --git a/cmd/ate-setup/internal/cmd/deploy_test.go b/cmd/ate-setup/internal/cmd/deploy_test.go index 31f91963db..c0074a724d 100644 --- a/cmd/ate-setup/internal/cmd/deploy_test.go +++ b/cmd/ate-setup/internal/cmd/deploy_test.go @@ -14,7 +14,14 @@ package cmd -import "testing" +import ( + "testing" + + "github.com/spf13/pflag" + + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/steps" +) // An upgrade or a rollback moves the control plane one component at a time, // so the components that only deploy ate-system used to install have their own @@ -42,3 +49,92 @@ func TestComponentDeploySubcommands(t *testing.T) { }) } } + +func resetCommandFlags(t *testing.T, flags *pflag.FlagSet) { + t.Helper() + reset := func() { + flags.VisitAll(func(f *pflag.Flag) { + _ = f.Value.Set(f.DefValue) + f.Changed = false + }) + } + reset() + t.Cleanup(reset) +} + +func TestDeploySandboxClassesFlag(t *testing.T) { + t.Run("deploy ate-system", func(t *testing.T) { + resetCommandFlags(t, deployAteSystemCmd.Flags()) + if err := deployAteSystemCmd.ParseFlags([]string{"--sandbox-classes=gvisor"}); err != nil { + t.Fatalf("ParseFlags(--sandbox-classes=gvisor): %v", err) + } + if err := deployAteSystemCmd.Args(deployAteSystemCmd, nil); err != nil { + t.Fatalf("Args(--sandbox-classes=gvisor): %v", err) + } + r, err := config.Resolve(deployAteSystemCmd.Flags(), config.ResolveOptions{}) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + cfg := &config.Config{} + cfg.SetResolved(r) + + prev := env + env = &steps.Env{Cfg: cfg} + t.Cleanup(func() { env = prev }) + + opts, err := deployOptions() + if err != nil { + t.Fatalf("deployOptions(): %v", err) + } + if opts.SandboxClasses != "gvisor" { + t.Errorf("opts.SandboxClasses = %q, want %q", opts.SandboxClasses, "gvisor") + } + }) + + t.Run("deploy sandboxconfig", func(t *testing.T) { + if deploySandboxConfigCmd.Flags().Lookup("sandbox-classes") == nil { + t.Fatal("deploy sandboxconfig is missing --sandbox-classes flag") + } + resetCommandFlags(t, deploySandboxConfigCmd.Flags()) + if err := deploySandboxConfigCmd.ParseFlags([]string{"--sandbox-classes=microvm"}); err != nil { + t.Fatalf("ParseFlags(--sandbox-classes=microvm): %v", err) + } + if err := deploySandboxConfigCmd.Args(deploySandboxConfigCmd, nil); err != nil { + t.Fatalf("Args(--sandbox-classes=microvm): %v", err) + } + r, err := config.Resolve(deploySandboxConfigCmd.Flags(), config.ResolveOptions{}) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + cfg := &config.Config{} + cfg.SetResolved(r) + + prev := env + env = &steps.Env{Cfg: cfg} + t.Cleanup(func() { env = prev }) + + got, err := sandboxClassesOption() + if err != nil { + t.Fatalf("sandboxClassesOption(): %v", err) + } + if got != "microvm" { + t.Errorf("sandboxClassesOption() = %q, want %q", got, "microvm") + } + }) + + t.Run("unknown class is rejected before connecting", func(t *testing.T) { + for _, args := range [][]string{ + {"deploy", "ate-system", "--sandbox-classes=kata"}, + {"deploy", "sandboxconfig", "--sandbox-classes=kata"}, + } { + resetCommandFlags(t, deployAteSystemCmd.Flags()) + resetCommandFlags(t, deploySandboxConfigCmd.Flags()) + if err := runRoot(t, args...); err == nil { + t.Errorf("runRoot(%v) = nil, want unknown sandbox class error", args) + } + if env != nil { + t.Errorf("runRoot(%v) connected to a cluster before rejecting unknown sandbox class", args) + } + } + }) +} diff --git a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go index 52a5036e62..3a264c0165 100644 --- a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go +++ b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go @@ -14,9 +14,7 @@ // Package countermicrovm installs the micro-VM variant of the counter demo. // -// It needs the cluster-wide `microvm` SandboxConfig that -// hack/install-microvm-deps.sh --install creates. It has no external-volume -// option: the CSI path is covered by demo-counter. +// It has no external-volume option: the CSI path is covered by demo-counter. package countermicrovm import ( @@ -32,7 +30,7 @@ const namespace = "ate-demo-counter-microvm" func init() { demos.Register(&demos.Substrate{ DemoName: "demo-counter-microvm", - Short: "The counter demo on micro-VM workers (needs hack/install-microvm-deps.sh --install)", + Short: "The counter demo on microVM workers", WorkerPoolManifest: "demos/counter/counter-microvm.yaml.tmpl", Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "counter-microvm"}}, Templates: []demos.SubstrateTemplate{{ diff --git a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go index 7c8146f494..d5b9276bba 100644 --- a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go +++ b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go @@ -14,12 +14,10 @@ // Package egressmicrovm installs the micro-VM variant of the egress demo. // -// It needs the cluster-wide `microvm` SandboxConfig that -// hack/install-microvm-deps.sh --install creates. It is a demo of its own -// rather than a flag on demo-egress so that it appears in help and in the -// `delete all` sweep, and so that both can be installed side by side: the -// networking e2e suite runs against whichever the sandbox class under test -// selects. +// It is a demo of its own rather than a flag on demo-egress so that it appears +// in help and in the `delete all` sweep, and so that both can be installed side +// by side: the networking e2e suite runs against whichever the sandbox class +// under test selects. package egressmicrovm import ( @@ -35,7 +33,7 @@ const namespace = "ate-demo-egress-microvm" func init() { demos.Register(&demos.Substrate{ DemoName: "demo-egress-microvm", - Short: "Egress policy enforcement on micro-VM workers (needs hack/install-microvm-deps.sh --install)", + Short: "Egress policy enforcement on microVM workers", WorkerPoolManifest: "demos/egress/egress-microvm.yaml.tmpl", Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "egress-microvm"}}, Templates: []demos.SubstrateTemplate{{ diff --git a/cmd/ate-setup/internal/kube/wait.go b/cmd/ate-setup/internal/kube/wait.go index d14b0b54cd..e3b31dd689 100644 --- a/cmd/ate-setup/internal/kube/wait.go +++ b/cmd/ate-setup/internal/kube/wait.go @@ -16,12 +16,14 @@ package kube import ( "context" + "errors" "fmt" "time" "go.ate.dev/substrate/cmd/ate-setup/internal/log" "go.ate.dev/substrate/internal/clustertrustbundle" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -183,6 +185,42 @@ func (c *Client) WaitNamespaceActive(ctx context.Context, name string, timeout t return nil } +// WaitJobComplete blocks until the named Job reports Complete=True, replacing +// `kubectl wait --for=condition=complete job/`. It fails immediately if +// the Job reports Failed=True. +func (c *Client) WaitJobComplete(ctx context.Context, namespace, name string, timeout time.Duration) error { + defer log.Elapsed(time.Now(), fmt.Sprintf("wait for job/%s", name)) + err := poll(ctx, timeout, func(ctx context.Context) (bool, error) { + job, err := c.Typed.BatchV1().Jobs(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + for _, cond := range job.Status.Conditions { + if cond.Type == batchv1.JobComplete && cond.Status == corev1.ConditionTrue { + return true, nil + } + if cond.Type == batchv1.JobFailed && cond.Status == corev1.ConditionTrue { + detail := cond.Message + if detail == "" { + detail = cond.Reason + } + if detail != "" { + return false, fmt.Errorf("job failed: %s", detail) + } + return false, errors.New("job failed") + } + } + return false, nil + }) + if err != nil { + return fmt.Errorf("waiting for job %s/%s to complete: %w", namespace, name, err) + } + return nil +} + // WaitCondition blocks until a custom resource reports the named status // condition as True, replacing `kubectl wait --for=condition=`. func (c *Client) WaitCondition(ctx context.Context, gvk schema.GroupVersionKind, namespace, name, condition string, timeout time.Duration) error { diff --git a/cmd/ate-setup/internal/kube/wait_test.go b/cmd/ate-setup/internal/kube/wait_test.go index 484e6ece5a..e85d4ea4ae 100644 --- a/cmd/ate-setup/internal/kube/wait_test.go +++ b/cmd/ate-setup/internal/kube/wait_test.go @@ -25,14 +25,18 @@ import ( "testing" "time" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" kubefake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" + k8stesting "k8s.io/client-go/testing" ) // shortPollInterval makes the retry path observable without a real sleep. @@ -184,3 +188,74 @@ func TestWaitClusterTrustBundlesVersions(t *testing.T) { type bundleRoundTripper func(*http.Request) (*http.Response, error) func (f bundleRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestWaitJobComplete(t *testing.T) { + shortPollInterval(t) + jobsResource := schema.GroupResource{Group: "batch", Resource: "jobs"} + + t.Run("retries transient error and NotFound until JobComplete", func(t *testing.T) { + kc := kubefake.NewSimpleClientset() + calls := 0 + kc.PrependReactor("get", "jobs", func(k8stesting.Action) (bool, runtime.Object, error) { + calls++ + switch calls { + case 1: + return true, nil, apierrors.NewServiceUnavailable("apiserver is starting") + case 2: + return true, nil, apierrors.NewNotFound(jobsResource, "rustfs-bucket-init") + default: + return true, &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ate-system", Name: "rustfs-bucket-init"}, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + {Type: batchv1.JobComplete, Status: corev1.ConditionTrue}, + }, + }, + }, nil + } + }) + c := &Client{Typed: kc} + if err := c.WaitJobComplete(t.Context(), "ate-system", "rustfs-bucket-init", time.Minute); err != nil { + t.Fatalf("WaitJobComplete() = %v, want nil", err) + } + if calls != 3 { + t.Errorf("Get called %d times, want 3", calls) + } + }) + + t.Run("aborts immediately when JobFailed is True", func(t *testing.T) { + kc := kubefake.NewSimpleClientset(&batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ate-system", Name: "rustfs-bucket-init"}, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + {Type: batchv1.JobFailed, Status: corev1.ConditionTrue, Message: "BackoffLimitExceeded"}, + }, + }, + }) + c := &Client{Typed: kc} + err := c.WaitJobComplete(t.Context(), "ate-system", "rustfs-bucket-init", time.Minute) + if err == nil || !strings.Contains(err.Error(), "BackoffLimitExceeded") { + t.Fatalf("WaitJobComplete() = %v, want BackoffLimitExceeded error", err) + } + if got := len(kc.Actions()); got != 1 { + t.Errorf("got %d API actions, want 1", got) + } + }) + + t.Run("aborts immediately on permanent API error", func(t *testing.T) { + kc := kubefake.NewSimpleClientset() + calls := 0 + kc.PrependReactor("get", "jobs", func(k8stesting.Action) (bool, runtime.Object, error) { + calls++ + return true, nil, apierrors.NewForbidden(jobsResource, "rustfs-bucket-init", errors.New("nope")) + }) + c := &Client{Typed: kc} + err := c.WaitJobComplete(t.Context(), "ate-system", "rustfs-bucket-init", time.Minute) + if !apierrors.IsForbidden(err) { + t.Fatalf("WaitJobComplete() = %v, want wrapped Forbidden error", err) + } + if calls != 1 { + t.Errorf("Get called %d times, want 1", calls) + } + }) +} diff --git a/cmd/ate-setup/internal/steps/benchmarks.go b/cmd/ate-setup/internal/steps/benchmarks.go index 68c51d020c..8025488eb1 100644 --- a/cmd/ate-setup/internal/steps/benchmarks.go +++ b/cmd/ate-setup/internal/steps/benchmarks.go @@ -25,14 +25,11 @@ import ( "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) -// The benchmark and micro-VM stacks are still driven by shell. They orchestrate -// several other scripts (image builds, asset assembly, object-store staging) -// that are out of scope for this command, so ate-setup shells out to them with -// a translated argument list rather than reimplementing them. -const ( - deployLocustScript = "benchmarking/deploy_locust.sh" - installMicrovmDepScript = "hack/install-microvm-deps.sh" -) +// The benchmark stack is still driven by shell. It orchestrates image builds +// and load-test deployment that are out of scope for this command, so +// ate-setup shells out to it with a translated argument list rather than +// reimplementing it. +const deployLocustScript = "benchmarking/deploy_locust.sh" // BenchmarkOptions shapes the benchmark WorkerPool. type BenchmarkOptions struct { @@ -69,15 +66,6 @@ func (e *Env) DeployBenchmarks(ctx context.Context, opts BenchmarkOptions) error return err } log.Stepf("deploy_benchmarks (worker_count=%d, sandbox_class=%s)", opts.WorkerCount, opts.SandboxClass) - - // The microvm SandboxConfig lives outside the default set installed by - // `deploy ate-system`, which only installs gvisor-default. The workloads - // deploy references it by name and would fail if this were skipped. - if opts.SandboxClass == config.SandboxClassMicrovm { - if err := e.runScript(ctx, installMicrovmDepScript, "--install"); err != nil { - return err - } - } return e.runScript(ctx, deployLocustScript, deployLocustArgs(opts, e.Cfg.OtlpEndpoint, e.Cfg.BenchmarkActorMemory)...) } @@ -109,18 +97,7 @@ func (e *Env) DeleteBenchmarks(ctx context.Context, opts BenchmarkOptions) error return err } log.Stepf("delete_benchmarks (sandbox_class=%s)", opts.SandboxClass) - - if err := e.runScript(ctx, deployLocustScript, "--delete"); err != nil { - return err - } - // The microvm SandboxConfig is cluster-wide. It is only torn down when - // the caller opted into microvm, and only after the benchmark pool is - // gone. The script leaves it in place if another pool still uses it. - // `delete all` tries again once every demo is gone. - if opts.SandboxClass == config.SandboxClassMicrovm { - return e.runScript(ctx, installMicrovmDepScript, "--delete") - } - return nil + return e.runScript(ctx, deployLocustScript, "--delete") } // runScript executes a repository script from the repository root with the diff --git a/cmd/ate-setup/internal/steps/delete.go b/cmd/ate-setup/internal/steps/delete.go index 7223e8a332..3570bbfcac 100644 --- a/cmd/ate-setup/internal/steps/delete.go +++ b/cmd/ate-setup/internal/steps/delete.go @@ -207,38 +207,14 @@ type Deleter interface { Delete(ctx context.Context, e *Env) error } -// DeleteAll removes every demo, then the microvm SandboxConfig, then the -// control plane. +// DeleteAll removes every registered demo and then the control plane. func (e *Env) DeleteAll(ctx context.Context, demos []Deleter) error { log.Step("delete_all") - if err := e.deleteDemos(ctx, demos); err != nil { - return err - } - return e.DeleteAteSystem(ctx) -} - -// deleteDemos removes every demo, then tries to delete the microvm -// SandboxConfig. An earlier `delete benchmarks` or `delete demo` may have -// left the config behind because a pool still used it. After the demos, no -// pool does. -// -// The delete is best effort. The script skips a config that is still in use, -// and a failure is only logged, because DeleteAteSystem deletes the config -// with the CRD anyway. -// -// The gvisor config needs no such step. It is installed and deleted with the -// control plane, and DeleteAteSystem strips its finalizer itself. The microvm -// config is managed by hack/install-microvm-deps.sh, whose `kubectl delete` -// would block while a pool still uses it. -func (e *Env) deleteDemos(ctx context.Context, demos []Deleter) error { for _, demo := range demos { if err := demo.Delete(ctx, e); err != nil { return err } } - if err := e.runScript(ctx, installMicrovmDepScript, "--delete"); err != nil { - log.Warnf("while deleting the microvm SandboxConfig: %v", err) - } - return nil + return e.DeleteAteSystem(ctx) } diff --git a/cmd/ate-setup/internal/steps/delete_test.go b/cmd/ate-setup/internal/steps/delete_test.go index 9274cadc6f..a255a23b70 100644 --- a/cmd/ate-setup/internal/steps/delete_test.go +++ b/cmd/ate-setup/internal/steps/delete_test.go @@ -18,8 +18,6 @@ import ( "context" "errors" "fmt" - "os" - "path/filepath" "slices" "strings" "testing" @@ -37,7 +35,6 @@ import ( "k8s.io/client-go/util/retry" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" - "go.ate.dev/substrate/cmd/ate-setup/internal/config" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/internal/clustertrustbundle" ) @@ -193,92 +190,6 @@ func TestReleaseSandboxConfigs(t *testing.T) { } } -// recordingDeleter is a Deleter that appends its name to a log file shared -// with the fake microvm script, so a test can read back the order of calls. -type recordingDeleter struct { - name string - log string - err error -} - -func (d recordingDeleter) Delete(context.Context, *Env) error { - appendLine(d.log, d.name) - return d.err -} - -func appendLine(path, line string) { - f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil { - panic(err) - } - defer f.Close() - fmt.Fprintln(f, line) -} - -func TestDeleteDemos(t *testing.T) { - demoErr := errors.New("demo failed") - tests := []struct { - name string - // demos are the deleters in order. A nil entry's err is nil. - demos []recordingDeleter - // scriptExit is the exit code of the fake install-microvm-deps.sh. - scriptExit int - wantCalls []string - wantErr error - }{ - { - name: "microvm SandboxConfig is deleted after every demo", - demos: []recordingDeleter{{name: "demo-a"}, {name: "demo-b"}}, - wantCalls: []string{"demo-a", "demo-b", "script --delete"}, - }, - { - name: "script failure is logged, not returned", - demos: []recordingDeleter{{name: "demo-a"}}, - scriptExit: 1, - wantCalls: []string{"demo-a", "script --delete"}, - }, - { - name: "a failing demo stops before the script", - demos: []recordingDeleter{{name: "demo-a", err: demoErr}, {name: "demo-b"}}, - wantCalls: []string{"demo-a"}, - wantErr: demoErr, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - root := t.TempDir() - calls := filepath.Join(root, "calls.log") - script := filepath.Join(root, installMicrovmDepScript) - if err := os.MkdirAll(filepath.Dir(script), 0o755); err != nil { - t.Fatal(err) - } - body := fmt.Sprintf("#!/bin/sh\necho \"script $*\" >> %q\nexit %d\n", calls, tt.scriptExit) - if err := os.WriteFile(script, []byte(body), 0o755); err != nil { - t.Fatal(err) - } - demos := make([]Deleter, 0, len(tt.demos)) - for _, d := range tt.demos { - d.log = calls - demos = append(demos, d) - } - e := &Env{Cfg: &config.Config{Root: root}} - - err := e.deleteDemos(t.Context(), demos) - if !errors.Is(err, tt.wantErr) { - t.Fatalf("deleteDemos() error = %v, want %v", err, tt.wantErr) - } - got, _ := os.ReadFile(calls) - gotCalls := strings.Fields(strings.ReplaceAll(strings.TrimSpace(string(got)), "script --delete", "script_--delete")) - for i := range gotCalls { - gotCalls[i] = strings.ReplaceAll(gotCalls[i], "script_--delete", "script --delete") - } - if !slices.Equal(gotCalls, tt.wantCalls) { - t.Errorf("calls = %q, want %q", gotCalls, tt.wantCalls) - } - }) - } -} - // trustBundle returns a bundle for signer the way the API server requires it // to be named: the signer name with "/" replaced by ":", then a suffix. func trustBundle(signer, suffix string) *certsv1.ClusterTrustBundle { diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index e235e96b0a..ba986975ec 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -43,12 +43,18 @@ type DeployOptions struct { // SetupCSI additionally installs the CSI driver (nfs, hostpath, both, none). // Kind only. The hostpath driver is Kind only. SetupCSI string + // SandboxClasses selects the default SandboxConfigs to install + // (comma-separated: gvisor, microvm; empty defaults to both). + SandboxClasses string } // Validate checks the options that can be checked without configuration or a // cluster, so the command can reject them while cobra is still parsing. func (o DeployOptions) Validate() error { - return ValidateCSIDriver(o.SetupCSI) + if err := ValidateCSIDriver(o.SetupCSI); err != nil { + return err + } + return ValidateSandboxClasses(o.SandboxClasses) } // DeployAteSystem installs the whole control plane: CRDs, RBAC, the @@ -82,6 +88,10 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err := e.CheckCSIDriver(opts.SetupCSI); err != nil { return err } + gvisor, microvm, err := parseSandboxClasses(opts.SandboxClasses) + if err != nil { + return err + } // The snapshot location is only written into ate-api-server's ConfigMap // partway through, so check it before the cluster is touched. if _, err := clusterSnapshotLocation(e.Cfg.SnapshotLocation, e.Cfg.PreviewGates); err != nil { @@ -90,6 +100,13 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err := e.requireNoWorkerPods(ctx); err != nil { return err } + var microvmPrep *microVMPrepared + if microvm { + microvmPrep, err = e.prepareMicroVMDeps(ctx) + if err != nil { + return err + } + } // The namespace has to exist before RBAC or CRDs are applied. if err := e.EnsureAteSystemNamespace(ctx); err != nil { @@ -102,6 +119,9 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err := e.DeployCRDs(ctx); err != nil { return err } + if err := e.applyInitialSandboxConfigs(ctx, gvisor); err != nil { + return err + } if err := e.EnsureAPIServerPrerequisites(ctx); err != nil { return err @@ -116,10 +136,6 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } - if err := e.DeploySandboxConfig(ctx); err != nil { - return err - } - // Ahead of the bundle below, for the same reason as the namespace: every // workload pulls this ConfigMap in via envFrom, and a container whose // envFrom target is missing will not start. @@ -174,6 +190,11 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err := e.applyAtenetEgress(ctx, provider); err != nil { return err } + if microvmPrep != nil { + if err := e.applyMicroVMSandboxConfig(ctx, microvmPrep); err != nil { + return err + } + } log.Step("Waiting for ATE system components to be ready...") type rollout struct{ kind, name string } @@ -588,23 +609,46 @@ func (e *Env) DeployAtenet(ctx context.Context) error { return nil } -// DeploySandboxConfig applies the SandboxConfig admission policy, then the -// default gVisor SandboxConfig. -func (e *Env) DeploySandboxConfig(ctx context.Context) error { +// DeploySandboxConfig applies the SandboxConfig admission policy and the +// default SandboxConfigs selected by sandboxClasses. +func (e *Env) DeploySandboxConfig(ctx context.Context, sandboxClasses string) error { log.Step("deploy_sandboxconfig") + gvisor, microvm, err := parseSandboxClasses(sandboxClasses) + if err != nil { + return err + } + var microvmPrep *microVMPrepared + if microvm { + microvmPrep, err = e.prepareMicroVMDeps(ctx) + if err != nil { + return err + } + } + if err := e.EnsureCRDs(ctx); err != nil { return err } + if err := e.applyInitialSandboxConfigs(ctx, gvisor); err != nil { + return err + } + if microvmPrep != nil { + return e.applyMicroVMSandboxConfig(ctx, microvmPrep) + } + return nil +} +// applyInitialSandboxConfigs applies the SandboxConfig ValidatingAdmissionPolicy +// and, when gvisor is enabled, the cluster-wide default gVisor SandboxConfig. +func (e *Env) applyInitialSandboxConfigs(ctx context.Context, gvisor bool) error { // Enforce per-class SandboxConfig asset requirements. This is applied - // before any SandboxConfig so the config below is validated too. + // before any SandboxConfig so the configs below are validated too. if err := e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-validation.yaml")); err != nil { return err } - - // Install the cluster-wide default gVisor SandboxConfig that gVisor - // WorkerPools resolve by default unless they pin their own configRef. + if !gvisor { + return nil + } return e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-gvisor.yaml")) } diff --git a/cmd/ate-setup/internal/steps/microvm.go b/cmd/ate-setup/internal/steps/microvm.go new file mode 100644 index 0000000000..365c27eca2 --- /dev/null +++ b/cmd/ate-setup/internal/steps/microvm.go @@ -0,0 +1,760 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "archive/tar" + "bufio" + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "io/fs" + "net" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/klauspost/compress/zstd" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + + atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/render" +) + +const ( + // MicroVMStageTimeout bounds the wait for the in-cluster rustfs Deployment + // and bucket-init Job on Kind. + MicroVMStageTimeout = 300 * time.Second + + // DefaultSandboxClasses is the default value of --sandbox-classes. + DefaultSandboxClasses = "gvisor,microvm" + + defaultKataVersion = "4.3.1" + defaultCloudHypervisorVersion = "v53.0" + + assetStampFile = ".asset-versions" + defaultKindMicroVMBucket = "ate-snapshots" + maxGCSBucketNameLength = 63 + microvmAssetsPrefix = "microvm-assets" + microvmSandboxConfigTemplate = "manifests/ate-install/sandboxconfig-microvm.yaml.tmpl" + microvmSandboxConfigName = "microvm" + + // Keep in sync with manifests/ate-install/kind/rustfs.yaml. + rustfsAWSCLIImage = "amazon/aws-cli:2.38.5@sha256:c8156a0e74e13929a42d7e60f0d4e14e7d28d980cd21822505d4bff80e5460c1" + + kataSharePrefix = "opt/kata/share/kata-containers/" + kataKernelSymlink = "opt/kata/share/kata-containers/vmlinux.container" + kataRootfsSymlink = "opt/kata/share/kata-containers/kata-containers.img" + kataLibexecVirtiofsd = "opt/kata/libexec/virtiofsd" + + maxMicroVMAssetBytes int64 = 8 << 30 +) + +var ( + microvmArches = []string{"amd64", "arm64"} + microvmAssets = []string{"cloud-hypervisor", "virtiofsd", "vmlinux", "rootfs.img"} + + // microvmTemplateAssetFiles maps each asset key in + // manifests/ate-install/sandboxconfig-microvm.yaml.tmpl to the local file + // name assembled under bin/microvm-assets//. + microvmTemplateAssetFiles = map[string]string{ + "cloud-hypervisor": "cloud-hypervisor", + "virtiofsd": "virtiofsd", + "kata-kernel": "vmlinux", + "kata-image": "rootfs.img", + } +) + +// microvmReleaseBaseURL is overridden in unit tests to point at an httptest.Server. +var microvmReleaseBaseURL = "https://github.com" + +// ValidateSandboxClasses reports whether classes is a valid --sandbox-classes value. +func ValidateSandboxClasses(classes string) error { + _, _, err := parseSandboxClasses(classes) + return err +} + +func parseSandboxClasses(classes string) (gvisor, microvm bool, err error) { + if strings.TrimSpace(classes) == "" { + return true, true, nil + } + for _, raw := range strings.Split(classes, ",") { + switch c := strings.TrimSpace(raw); c { + case "gvisor": + gvisor = true + case "microvm": + microvm = true + default: + return false, false, fmt.Errorf("unknown sandbox class %q (valid options: gvisor, microvm)", c) + } + } + return gvisor, microvm, nil +} + +// microVMPrepared holds the validated target bucket and per-architecture +// asset digests assembled by prepareMicroVMDeps before the cluster is touched. +type microVMPrepared struct { + bucket string + digests map[string]map[string]string +} + +// microVMArches returns the architectures to assemble and stage: only the host +// architecture on Kind (where nodes always match runtime.GOARCH), and both +// amd64 and arm64 on non-Kind clusters. +func (e *Env) microVMArches() []string { + if e.Cfg != nil && e.Cfg.Kind { + return []string{runtime.GOARCH} + } + return microvmArches +} + +// prepareMicroVMDeps validates the microVM prerequisites, assembles the guest +// assets locally, and on non-Kind clusters uploads them to GCS before any +// cluster resources are modified. +func (e *Env) prepareMicroVMDeps(ctx context.Context) (*microVMPrepared, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + log.Step("prepare_microvm_deps") + + bucket, err := e.checkMicroVMDeps() + if err != nil { + return nil, err + } + digests, err := loadMicroVMAssetDigests(e.Cfg.Manifest("sandboxconfig-microvm.yaml.tmpl")) + if err != nil { + return nil, err + } + + for _, arch := range e.microVMArches() { + outDir := e.Cfg.Path("bin", microvmAssetsPrefix, arch) + wantStamp := microvmAssetStamp(arch, defaultKataVersion, defaultCloudHypervisorVersion) + needAssemble, stale := microvmAssetsNeedAssemble(outDir, wantStamp, digests[arch]) + if stale { + log.Infof("Asset set in %s is stale; re-assembling.", outDir) + } + if needAssemble { + log.Infof("Assembling microVM assets into %s (arch=%s)...", outDir, arch) + if err := assembleMicroVMAssets(ctx, outDir, arch, defaultKataVersion, defaultCloudHypervisorVersion, digests[arch]); err != nil { + return nil, err + } + } else { + log.Infof("Assets already present in %s; skipping assemble.", outDir) + } + } + + if !e.Cfg.Kind { + log.Infof("Uploading assets to gs://%s/%s/ ...", bucket, microvmAssetsPrefix) + if err := e.stageMicroVMAssetsToGCS(ctx, bucket, digests); err != nil { + return nil, err + } + } + return µVMPrepared{bucket: bucket, digests: digests}, nil +} + +// applyMicroVMSandboxConfig stages the assembled microVM guest assets into +// in-cluster rustfs on Kind and applies the cluster-wide `microvm` SandboxConfig. +func (e *Env) applyMicroVMSandboxConfig(ctx context.Context, prep *microVMPrepared) error { + if err := ctx.Err(); err != nil { + return err + } + log.Step("deploy_microvm_sandboxconfig") + + if e.Cfg.Kind { + log.Infof("Staging assets to in-cluster rustfs bucket %s (%s/)...", prep.bucket, microvmAssetsPrefix) + if err := e.stageMicroVMAssetsToRustfs(ctx, prep.bucket, prep.digests); err != nil { + return err + } + } + + log.Infof("Applying microvm SandboxConfig from %s...", microvmSandboxConfigTemplate) + rendered, err := render.Template( + e.Cfg.Manifest("sandboxconfig-microvm.yaml.tmpl"), + map[string]string{ + "BUCKET_SCHEME": e.microVMBucketScheme(), + "ASSET_BUCKET_NAME": prep.bucket, + }, + nil, + ) + if err != nil { + return err + } + if err := e.Kube.ApplyBytes(ctx, rendered); err != nil { + return err + } + log.Infof("Done. Installed cluster-wide microvm SandboxConfig.") + return nil +} + +// loadMicroVMAssetDigests reads the expected per-architecture SHA-256 digests +// for microvmAssets from the default version of sandboxconfig-microvm.yaml.tmpl. +func loadMicroVMAssetDigests(tmplPath string) (map[string]map[string]string, error) { + rendered, err := render.Template( + tmplPath, + map[string]string{ + "BUCKET_SCHEME": "gs", + "ASSET_BUCKET_NAME": "placeholder", + }, + nil, + ) + if err != nil { + return nil, err + } + objs, err := kube.DecodeManifestBytes(rendered) + if err != nil { + return nil, err + } + if len(objs) != 1 { + return nil, fmt.Errorf("%s decoded to %d objects, want 1", tmplPath, len(objs)) + } + var sc atev1.SandboxConfig + if err := k8sruntime.DefaultUnstructuredConverter.FromUnstructured(objs[0].Object, &sc); err != nil { + return nil, fmt.Errorf("decoding %s: %w", tmplPath, err) + } + defaultVer, ok := sc.Spec.DefaultVersionConfig() + if !ok { + return nil, fmt.Errorf("%s has no enabled version matching defaultVersion %q", tmplPath, sc.Spec.DefaultVersion) + } + + out := make(map[string]map[string]string, len(microvmArches)) + for _, arch := range microvmArches { + archAssets, ok := defaultVer.Assets[arch] + if !ok { + return nil, fmt.Errorf("%s default version %q has no assets for arch %q", tmplPath, defaultVer.Name, arch) + } + files := make(map[string]string, len(microvmTemplateAssetFiles)) + for tmplKey, fileName := range microvmTemplateAssetFiles { + af, ok := archAssets[tmplKey] + if !ok || af.SHA256 == "" { + return nil, fmt.Errorf("%s default version %q arch %q missing sha256 for %q", tmplPath, defaultVer.Name, arch, tmplKey) + } + wantSuffix := fmt.Sprintf("/%s/%s/%s", microvmAssetsPrefix, arch, af.SHA256) + if !strings.HasSuffix(af.URL, wantSuffix) { + return nil, fmt.Errorf("%s default version %q arch %q %q url %q does not end with %q", tmplPath, defaultVer.Name, arch, tmplKey, af.URL, wantSuffix) + } + files[fileName] = af.SHA256 + } + out[arch] = files + } + return out, nil +} + +// checkMicroVMDeps validates the asset bucket configuration and staging CLI +// tool before cluster mutation or archive downloads begin. +func (e *Env) checkMicroVMDeps() (string, error) { + bucket, err := e.microVMBucket() + if err != nil { + return "", err + } + if e.Cfg.Kind { + if _, err := exec.LookPath("docker"); err != nil { + return "", fmt.Errorf("'docker' is required to run the S3 client but was not found in PATH: %w", err) + } + } else { + if _, err := exec.LookPath("gcloud"); err != nil { + return "", fmt.Errorf("'gcloud' is required to upload microVM assets to GCS but was not found in PATH: %w", err) + } + } + return bucket, nil +} + +// microVMBucket returns the bucket where microVM guest assets are staged: the +// shared rustfs bucket on Kind, and -assets on GKE (where atelet +// has read access only to the asset bucket). +func (e *Env) microVMBucket() (string, error) { + if e.Cfg.Kind { + if e.Cfg.BucketName != "" { + return e.Cfg.BucketName, nil + } + return defaultKindMicroVMBucket, nil + } + if e.Cfg.BucketName == "" { + return "", fmt.Errorf("BUCKET_NAME (--storage-bucket-name or config.storage.bucketName) must be set to stage microVM assets on non-Kind clusters, or pass --sandbox-classes=gvisor") + } + assetBucket := e.Cfg.BucketName + "-assets" + if len(assetBucket) > maxGCSBucketNameLength { + return "", fmt.Errorf("bucket name %q is too long: the asset bucket %q would have %d characters, and GCS allows %d", e.Cfg.BucketName, assetBucket, len(assetBucket), maxGCSBucketNameLength) + } + return assetBucket, nil +} + +// microVMBucketScheme returns the URL scheme for the node's object-store +// plugin: s3 on Kind (rustfs) and gs on GKE. +func (e *Env) microVMBucketScheme() string { + if e.Cfg.Kind { + return "s3" + } + return "gs" +} + +func cloudHypervisorAssetName(arch string) (string, error) { + switch arch { + case "arm64": + return "cloud-hypervisor-static-aarch64", nil + case "amd64": + return "cloud-hypervisor-static", nil + default: + return "", fmt.Errorf("unsupported arch %q", arch) + } +} + +func microvmAssetStamp(arch, kataVer, chVer string) string { + return fmt.Sprintf("arch=%s\nkata=%s\ncloud-hypervisor=%s\n", + arch, kataVer, chVer) +} + +func checkMicroVMAssets(outDir string) error { + for _, f := range microvmAssets { + if fi, err := os.Stat(filepath.Join(outDir, f)); err != nil || !fi.Mode().IsRegular() { + return fmt.Errorf("missing asset %s/%s", outDir, f) + } + } + return nil +} + +func verifyMicroVMAssetDigests(outDir string, digests map[string]string) error { + if len(digests) == 0 { + return fmt.Errorf("no pinned microVM asset digests") + } + var mismatches []string + for _, f := range microvmAssets { + p := filepath.Join(outDir, f) + got, err := fileSHA256(p) + if err != nil { + return err + } + if want := digests[f]; got != want { + mismatches = append(mismatches, fmt.Sprintf("%s: got %s, want %s", f, got, want)) + } + } + if len(mismatches) > 0 { + return fmt.Errorf("microVM asset sha256 mismatch in %s (pinned in %s):\n %s", + outDir, microvmSandboxConfigTemplate, strings.Join(mismatches, "\n ")) + } + return nil +} + +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +func microvmAssetsNeedAssemble(outDir, wantStamp string, digests map[string]string) (needAssemble, stale bool) { + if err := checkMicroVMAssets(outDir); err != nil { + return true, false + } + got, err := os.ReadFile(filepath.Join(outDir, assetStampFile)) + if err != nil || string(got) != wantStamp { + return true, true + } + if err := verifyMicroVMAssetDigests(outDir, digests); err != nil { + return true, true + } + return false, false +} + +// assembleMicroVMAssets downloads the Kata static tarball and Cloud Hypervisor +// binary for arch into outDir, verifies the SHA-256 digests against digests, +// and writes the .asset-versions stamp last. +func assembleMicroVMAssets(ctx context.Context, outDir, arch, kataVer, chVer string, digests map[string]string) error { + chAsset, err := cloudHypervisorAssetName(arch) + if err != nil { + return err + } + if err := os.MkdirAll(outDir, 0o755); err != nil { + return fmt.Errorf("while creating %s: %w", outDir, err) + } + + // Remove any existing stamp before mutating files in outDir so a failed + // run leaves an unstamped directory that the next run will re-assemble. + stampPath := filepath.Join(outDir, assetStampFile) + _ = os.Remove(stampPath) + + workDir, err := os.MkdirTemp(outDir, ".assemble-*") + if err != nil { + return fmt.Errorf("while creating temporary directory in %s: %w", outDir, err) + } + defer func() { _ = os.RemoveAll(workDir) }() + + baseURL := strings.TrimSuffix(microvmReleaseBaseURL, "/") + kataURL := fmt.Sprintf("%s/kata-containers/kata-containers/releases/download/%s/kata-static-%s-%s.tar.zst", + baseURL, kataVer, kataVer, arch) + log.Infof(">> Downloading kata-static %s (%s)...", kataVer, arch) + if err := downloadAndExtractKataAssets(ctx, kataURL, workDir, outDir); err != nil { + return err + } + + chURL := fmt.Sprintf("%s/cloud-hypervisor/cloud-hypervisor/releases/download/%s/%s", baseURL, chVer, chAsset) + log.Infof(">> Downloading cloud-hypervisor %s (%s)...", chVer, chAsset) + chTmp := filepath.Join(workDir, "cloud-hypervisor") + if err := downloadToFile(ctx, chURL, chTmp, 0o755); err != nil { + return err + } + if err := os.Rename(chTmp, filepath.Join(outDir, "cloud-hypervisor")); err != nil { + return err + } + + if err := checkMicroVMAssets(outDir); err != nil { + return err + } + if err := verifyMicroVMAssetDigests(outDir, digests); err != nil { + return err + } + if err := os.WriteFile(stampPath, []byte(microvmAssetStamp(arch, kataVer, chVer)), 0o644); err != nil { + return fmt.Errorf("while writing %s: %w", stampPath, err) + } + log.Infof(">> Assets assembled in %s.", outDir) + return nil +} + +func downloadAndExtractKataAssets(ctx context.Context, kataURL, workDir, outDir string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, kataURL, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("while downloading %s: %w", kataURL, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("while downloading %s: unexpected HTTP status %s", kataURL, resp.Status) + } + + links, err := extractKataTarZst(ctx, resp.Body, workDir) + if err != nil { + return fmt.Errorf("while extracting %s: %w", kataURL, err) + } + + for _, item := range []struct { + archivePath string + outName string + mode fs.FileMode + }{ + {archivePath: kataKernelSymlink, outName: "vmlinux", mode: 0o644}, + // rootfs.img is attached over virtio-pmem and mounted with DAX (cmd/ateworker-microvm/run.go). + // The image must meet the following requirements: + // - a size that is a multiple of 2 MiB (cloud-hypervisor) + // - partition 1 aligned to the guest page size + // - a filesystem block size equal to the guest page size + // The kata-containers.img used here is built for this, and vmlinux.container uses 4 KiB pages. + // See https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder/image-builder/image_builder.sh + {archivePath: kataRootfsSymlink, outName: "rootfs.img", mode: 0o644}, + {archivePath: kataLibexecVirtiofsd, outName: "virtiofsd", mode: 0o755}, + } { + resolved := item.archivePath + for range 8 { + target, ok := links[resolved] + if !ok { + break + } + resolved = target + } + src := filepath.Join(workDir, filepath.FromSlash(resolved)) + if fi, err := os.Stat(src); err != nil || !fi.Mode().IsRegular() { + return fmt.Errorf("while resolving %s in kata archive: missing regular file %s", item.archivePath, resolved) + } + if err := os.Chmod(src, item.mode); err != nil { + return err + } + if err := os.Rename(src, filepath.Join(outDir, item.outName)); err != nil { + return err + } + } + return nil +} + +// extractKataTarZst decompresses a zstd-compressed tar stream, writes regular +// files under opt/kata/share/kata-containers/ and opt/kata/libexec/virtiofsd +// into workDir, and returns a map of symlink paths to their cleaned archive +// targets. +func extractKataTarZst(ctx context.Context, r io.Reader, workDir string) (map[string]string, error) { + zr, err := zstd.NewReader(bufio.NewReader(r)) + if err != nil { + return nil, fmt.Errorf("creating zstd reader: %w", err) + } + defer zr.Close() + + links := map[string]string{} + tr := tar.NewReader(zr) + var totalBytes int64 + for { + if err := ctx.Err(); err != nil { + return nil, err + } + hdr, err := tr.Next() + if err == io.EOF { + return links, nil + } + if err != nil { + return nil, err + } + + name := strings.TrimPrefix(path.Clean(hdr.Name), "/") + if name == "." || name == "" { + continue + } + if !filepath.IsLocal(filepath.FromSlash(name)) { + return nil, fmt.Errorf("tar entry %q escapes the extraction directory", hdr.Name) + } + if !strings.HasPrefix(name, kataSharePrefix) && name != kataLibexecVirtiofsd { + continue + } + + switch hdr.Typeflag { + case tar.TypeReg: + totalBytes += hdr.Size + if totalBytes > maxMicroVMAssetBytes { + return nil, fmt.Errorf("kata archive exceeds %d-byte limit", maxMicroVMAssetBytes) + } + dst := filepath.Join(workDir, filepath.FromSlash(name)) + if err := writeStreamToFile(dst, 0o600, tr); err != nil { + return nil, err + } + case tar.TypeSymlink: + if path.IsAbs(hdr.Linkname) { + return nil, fmt.Errorf("tar symlink %q has absolute target %q", hdr.Name, hdr.Linkname) + } + target := path.Clean(path.Join(path.Dir(name), hdr.Linkname)) + if !filepath.IsLocal(filepath.FromSlash(target)) { + return nil, fmt.Errorf("tar symlink %q target %q escapes the extraction directory", hdr.Name, hdr.Linkname) + } + links[name] = target + } + } +} + +func writeStreamToFile(dstPath string, mode fs.FileMode, r io.Reader) error { + if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { + return err + } + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return err + } + n, err := io.Copy(f, io.LimitReader(r, maxMicroVMAssetBytes+1)) + closeErr := f.Close() + if err != nil { + return err + } + if n > maxMicroVMAssetBytes { + return fmt.Errorf("extracted file exceeds %d-byte limit", maxMicroVMAssetBytes) + } + return closeErr +} + +func downloadToFile(ctx context.Context, url, dstPath string, mode fs.FileMode) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("while downloading %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("while downloading %s: unexpected HTTP status %s", url, resp.Status) + } + return writeStreamToFile(dstPath, mode, resp.Body) +} + +func (e *Env) stageMicroVMAssetsToRustfs(ctx context.Context, bucket string, digests map[string]map[string]string) error { + ns := e.Namespace() + timeout := e.Cfg.WaitTimeout(MicroVMStageTimeout) + log.Infof(">> Waiting for rustfs in namespace %s...", ns) + if err := e.Kube.RolloutStatus(ctx, kube.KindDeployment, ns, "rustfs", timeout); err != nil { + return err + } + if err := e.Kube.WaitJobComplete(ctx, ns, "rustfs-bucket-init", timeout); err != nil { + return err + } + + node, err := e.rustfsNodeName(ctx, ns) + if err != nil { + return err + } + svc, err := e.Kube.Typed.CoreV1().Services(ns).Get(ctx, "rustfs", metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("while getting service %s/rustfs: %w", ns, err) + } + if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == corev1.ClusterIPNone { + return fmt.Errorf("service %s/rustfs has no ClusterIP", ns) + } + endpoint := "http://" + net.JoinHostPort(svc.Spec.ClusterIP, "9000") + + // Credentials and region are fixed in manifests/ate-install/kind/rustfs.yaml. + awsCLIArgs := func(args ...string) []string { + return append([]string{ + "run", "--rm", "-i", + "--network", "container:" + node, + "-e", "AWS_ACCESS_KEY_ID=rustfsadmin", + "-e", "AWS_SECRET_ACCESS_KEY=rustfsadmin", + "-e", "AWS_REGION=us-east-1", + "-e", "AWS_ENDPOINT_URL=" + endpoint, + rustfsAWSCLIImage, + }, args...) + } + awsCLI := func(stdin io.Reader, args ...string) error { + return e.runCmd(ctx, stdin, "docker", awsCLIArgs(args...)...) + } + + // rustfs-bucket-init creates only defaultKindMicroVMBucket. + if bucket != defaultKindMicroVMBucket { + if err := awsCLI(nil, "s3api", "head-bucket", "--bucket", bucket); err != nil { + if err := awsCLI(nil, "s3api", "create-bucket", "--bucket", bucket); err != nil { + return fmt.Errorf("while creating bucket %s in rustfs: %w", bucket, err) + } + } + } + + prefixURL := fmt.Sprintf("s3://%s/%s/", bucket, microvmAssetsPrefix) + // s3 ls exits non-zero when the prefix has no objects yet; cp surfaces real errors. + lsOut, _ := e.runCmdOutput(ctx, "docker", awsCLIArgs("s3", "ls", "--recursive", prefixURL)...) + existing := fieldsSet(lsOut) + + log.Infof(">> Uploading assets to %s via %s (netns of %s)...", prefixURL, endpoint, node) + for _, arch := range e.microVMArches() { + outDir := e.Cfg.Path("bin", microvmAssetsPrefix, arch) + for _, f := range microvmAssets { + sha := digests[arch][f] + key := fmt.Sprintf("%s/%s/%s", microvmAssetsPrefix, arch, sha) + if existing[key] { + log.Infof(" %s/%s (%s) already in s3://%s; skipping.", arch, f, sha, bucket) + continue + } + log.Infof(" %s/%s -> %s", arch, f, key) + file, err := os.Open(filepath.Join(outDir, f)) + if err != nil { + return err + } + dst := fmt.Sprintf("s3://%s/%s", bucket, key) + err = awsCLI(file, "s3", "cp", "-", dst) + _ = file.Close() + if err != nil { + return fmt.Errorf("while uploading %s/%s to rustfs: %w", arch, f, err) + } + } + } + + log.Infof(">> Done. Verify:") + return awsCLI(nil, "s3", "ls", "--recursive", prefixURL) +} + +// rustfsNodeName returns the node hosting the rustfs pod so the S3 upload +// runs in that node's network namespace, which rustfs's NetworkPolicy admits. +func (e *Env) rustfsNodeName(ctx context.Context, ns string) (string, error) { + pods, err := e.Kube.Typed.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{ + LabelSelector: "app=rustfs", + }) + if err != nil { + return "", fmt.Errorf("while listing rustfs pods: %w", err) + } + for _, p := range pods.Items { + if p.DeletionTimestamp != nil || p.Status.Phase == corev1.PodSucceeded || p.Status.Phase == corev1.PodFailed { + continue + } + if p.Spec.NodeName != "" { + return p.Spec.NodeName, nil + } + } + return e.kindNodeName(ctx) +} + +func (e *Env) stageMicroVMAssetsToGCS(ctx context.Context, bucket string, digests map[string]map[string]string) error { + var projectFlag []string + if e.Cfg.ProjectID != "" { + projectFlag = []string{"--project=" + e.Cfg.ProjectID} + } + + prefixURL := fmt.Sprintf("gs://%s/%s/", bucket, microvmAssetsPrefix) + lsArgs := append(append([]string{"storage", "ls", "--recursive"}, projectFlag...), prefixURL) + // gcloud storage ls exits non-zero when the prefix has no objects yet; cp surfaces real errors. + lsOut, _ := e.runCmdOutput(ctx, "gcloud", lsArgs...) + existing := fieldsSet(lsOut) + + for _, arch := range e.microVMArches() { + outDir := e.Cfg.Path("bin", microvmAssetsPrefix, arch) + for _, f := range microvmAssets { + sha := digests[arch][f] + dst := fmt.Sprintf("gs://%s/%s/%s/%s", bucket, microvmAssetsPrefix, arch, sha) + if existing[dst] { + log.Infof(" %s/%s (%s) already in gs://%s; skipping.", arch, f, sha, bucket) + continue + } + log.Infof(" %s/%s -> %s", arch, f, dst) + args := append(append([]string{"storage", "cp"}, projectFlag...), filepath.Join(outDir, f), dst) + if err := e.runCmd(ctx, nil, "gcloud", args...); err != nil { + return fmt.Errorf("while uploading %s/%s to GCS: %w", arch, f, err) + } + } + } + + log.Infof(">> Done. Verify:") + return e.runCmd(ctx, nil, "gcloud", lsArgs...) +} + +func fieldsSet(s string) map[string]bool { + fields := strings.Fields(s) + out := make(map[string]bool, len(fields)) + for _, f := range fields { + out[f] = true + } + return out +} + +func (e *Env) runCmd(ctx context.Context, stdin io.Reader, name string, args ...string) error { + cmd := exec.CommandContext(ctx, name, args...) + if e.Cfg != nil { + cmd.Dir = e.Cfg.Root + cmd.Env = e.Cfg.ScriptEnv() + } + cmd.Stdin = stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func (e *Env) runCmdOutput(ctx context.Context, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + if e.Cfg != nil { + cmd.Dir = e.Cfg.Root + cmd.Env = e.Cfg.ScriptEnv() + } + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = io.Discard + err := cmd.Run() + return out.String(), err +} diff --git a/cmd/ate-setup/internal/steps/microvm_test.go b/cmd/ate-setup/internal/steps/microvm_test.go new file mode 100644 index 0000000000..c62bb87385 --- /dev/null +++ b/cmd/ate-setup/internal/steps/microvm_test.go @@ -0,0 +1,824 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" + "go.ate.dev/substrate/cmd/ate-setup/internal/render" +) + +func silenceStepLog(t *testing.T) { + t.Helper() + log.SetOutput(io.Discard) + t.Cleanup(func() { log.SetOutput(os.Stdout) }) +} + +func sha256HexString(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +func testAssetDigests() map[string]string { + return map[string]string{ + "cloud-hypervisor": sha256HexString("ch-bytes"), + "virtiofsd": sha256HexString("virtiofsd-bytes"), + "vmlinux": sha256HexString("vmlinux-bytes"), + "rootfs.img": sha256HexString("rootfs-bytes"), + } +} + +func writeValidTestAssets(t *testing.T, dir string) { + t.Helper() + for name, body := range map[string]string{ + "cloud-hypervisor": "ch-bytes", + "virtiofsd": "virtiofsd-bytes", + "vmlinux": "vmlinux-bytes", + "rootfs.img": "rootfs-bytes", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } +} + +func TestVerifyMicroVMAssetDigests(t *testing.T) { + digests := testAssetDigests() + + t.Run("matching digests succeed", func(t *testing.T) { + dir := t.TempDir() + writeValidTestAssets(t, dir) + if err := verifyMicroVMAssetDigests(dir, digests); err != nil { + t.Fatalf("verifyMicroVMAssetDigests() = %v, want nil", err) + } + }) + + t.Run("mismatched digests report all mismatches at once", func(t *testing.T) { + dir := t.TempDir() + writeValidTestAssets(t, dir) + if err := os.WriteFile(filepath.Join(dir, "vmlinux"), []byte("bad-kernel"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "rootfs.img"), []byte("bad-rootfs"), 0o644); err != nil { + t.Fatal(err) + } + err := verifyMicroVMAssetDigests(dir, digests) + if err == nil { + t.Fatal("verifyMicroVMAssetDigests() = nil, want mismatch error") + } + for _, wantSubstr := range []string{ + "vmlinux: got " + sha256HexString("bad-kernel") + ", want " + digests["vmlinux"], + "rootfs.img: got " + sha256HexString("bad-rootfs") + ", want " + digests["rootfs.img"], + } { + if !strings.Contains(err.Error(), wantSubstr) { + t.Errorf("verifyMicroVMAssetDigests() = %q, missing %q", err, wantSubstr) + } + } + }) + + t.Run("missing file fails", func(t *testing.T) { + dir := t.TempDir() + writeValidTestAssets(t, dir) + if err := os.Remove(filepath.Join(dir, "rootfs.img")); err != nil { + t.Fatal(err) + } + if err := verifyMicroVMAssetDigests(dir, digests); err == nil { + t.Fatal("verifyMicroVMAssetDigests() succeeded with missing file, want error") + } + }) + + t.Run("empty digests map fails", func(t *testing.T) { + if err := verifyMicroVMAssetDigests(t.TempDir(), nil); err == nil { + t.Fatal("verifyMicroVMAssetDigests() succeeded with nil digests, want error") + } + }) +} + +func TestMicroVMAssetsNeedAssemble(t *testing.T) { + stamp := microvmAssetStamp("amd64", defaultKataVersion, defaultCloudHypervisorVersion) + const wantStamp = "arch=amd64\nkata=4.3.1\ncloud-hypervisor=v53.0\n" + if stamp != wantStamp { + t.Fatalf("microvmAssetStamp() =\n%q\nwant\n%q", stamp, wantStamp) + } + digests := testAssetDigests() + + t.Run("missing file triggers assemble without stale warning", func(t *testing.T) { + dir := t.TempDir() + for _, f := range microvmAssets[:3] { + if err := os.WriteFile(filepath.Join(dir, f), []byte(f), 0o644); err != nil { + t.Fatal(err) + } + } + need, stale := microvmAssetsNeedAssemble(dir, stamp, digests) + if !need || stale { + t.Errorf("microvmAssetsNeedAssemble() = (%v, %v), want (true, false)", need, stale) + } + }) + + t.Run("missing stamp with all files present reports stale", func(t *testing.T) { + dir := t.TempDir() + writeValidTestAssets(t, dir) + need, stale := microvmAssetsNeedAssemble(dir, stamp, digests) + if !need || !stale { + t.Errorf("microvmAssetsNeedAssemble() = (%v, %v), want (true, true)", need, stale) + } + }) + + t.Run("mismatched stamp reports stale", func(t *testing.T) { + dir := t.TempDir() + writeValidTestAssets(t, dir) + oldStamp := microvmAssetStamp("amd64", "4.0.0", defaultCloudHypervisorVersion) + if err := os.WriteFile(filepath.Join(dir, assetStampFile), []byte(oldStamp), 0o644); err != nil { + t.Fatal(err) + } + need, stale := microvmAssetsNeedAssemble(dir, stamp, digests) + if !need || !stale { + t.Errorf("microvmAssetsNeedAssemble() = (%v, %v), want (true, true)", need, stale) + } + }) + + t.Run("matching stamp with corrupted file digest reports stale", func(t *testing.T) { + dir := t.TempDir() + writeValidTestAssets(t, dir) + if err := os.WriteFile(filepath.Join(dir, assetStampFile), []byte(stamp), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "vmlinux"), []byte("corrupted"), 0o644); err != nil { + t.Fatal(err) + } + need, stale := microvmAssetsNeedAssemble(dir, stamp, digests) + if !need || !stale { + t.Errorf("microvmAssetsNeedAssemble() = (%v, %v), want (true, true)", need, stale) + } + }) + + t.Run("matching stamp and matching digests skip assemble", func(t *testing.T) { + dir := t.TempDir() + writeValidTestAssets(t, dir) + if err := os.WriteFile(filepath.Join(dir, assetStampFile), []byte(stamp), 0o644); err != nil { + t.Fatal(err) + } + need, stale := microvmAssetsNeedAssemble(dir, stamp, digests) + if need || stale { + t.Errorf("microvmAssetsNeedAssemble() = (%v, %v), want (false, false)", need, stale) + } + }) +} + +type tarTestEntry struct { + name string + typeflag byte + linkname string + body string + mode int64 +} + +func buildKataTarZst(t *testing.T, entries []tarTestEntry) []byte { + t.Helper() + var buf bytes.Buffer + zw, err := zstd.NewWriter(&buf) + if err != nil { + t.Fatal(err) + } + tw := tar.NewWriter(zw) + for _, e := range entries { + mode := e.mode + if mode == 0 { + mode = 0o644 + } + hdr := &tar.Header{ + Name: e.name, + Typeflag: e.typeflag, + Linkname: e.linkname, + Size: int64(len(e.body)), + Mode: mode, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if e.typeflag == tar.TypeReg { + if _, err := tw.Write([]byte(e.body)); err != nil { + t.Fatal(err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func validKataArchive(t *testing.T, virtiofsdScript string) []byte { + t.Helper() + // Place the vmlinux.container symlink BEFORE its target file to verify + // extraction does not depend on entry order in the tar stream. + return buildKataTarZst(t, []tarTestEntry{ + { + name: "opt/kata/share/kata-containers/vmlinux.container", + typeflag: tar.TypeSymlink, + linkname: "vmlinux-6.12.47-173", + }, + { + name: "opt/kata/share/kata-containers/vmlinux-6.12.47-173", + typeflag: tar.TypeReg, + body: "fake-kernel-bytes", + }, + { + name: "opt/kata/share/kata-containers/kata-containers-ubuntu.img", + typeflag: tar.TypeReg, + body: "fake-rootfs-bytes", + }, + { + name: "opt/kata/share/kata-containers/kata-containers.img", + typeflag: tar.TypeSymlink, + linkname: "kata-containers-ubuntu.img", + }, + { + name: "opt/kata/libexec/virtiofsd", + typeflag: tar.TypeReg, + body: virtiofsdScript, + mode: 0o755, + }, + { + name: "opt/kata/bin/qemu-system-x86_64", + typeflag: tar.TypeReg, + body: "ignored-qemu-binary", + }, + }) +} + +func serveFakeReleases(t *testing.T, archive []byte, chBody string, failCH bool) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/kata-containers/kata-containers/releases/download/4.3.1/kata-static-4.3.1-amd64.tar.zst": + _, _ = w.Write(archive) + case "/cloud-hypervisor/cloud-hypervisor/releases/download/v53.0/cloud-hypervisor-static": + if failCH { + w.Header().Set("Content-Length", "100") + _, _ = w.Write([]byte("partial")) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + panic(http.ErrAbortHandler) + } + _, _ = w.Write([]byte(chBody)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + origURL := microvmReleaseBaseURL + microvmReleaseBaseURL = srv.URL + t.Cleanup(func() { microvmReleaseBaseURL = origURL }) +} + +func TestAssembleMicroVMAssets(t *testing.T) { + silenceStepLog(t) + + const virtiofsdBody = "fake-virtiofsd-binary" + const chBody = "fake-cloud-hypervisor-binary" + digests := map[string]string{ + "cloud-hypervisor": sha256HexString(chBody), + "virtiofsd": sha256HexString(virtiofsdBody), + "vmlinux": sha256HexString("fake-kernel-bytes"), + "rootfs.img": sha256HexString("fake-rootfs-bytes"), + } + + serveFakeReleases(t, validKataArchive(t, virtiofsdBody), chBody, false) + + outDir := t.TempDir() + if err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.3.1", "v53.0", digests); err != nil { + t.Fatalf("assembleMicroVMAssets() = %v", err) + } + + for name, wantBody := range map[string]string{ + "vmlinux": "fake-kernel-bytes", + "rootfs.img": "fake-rootfs-bytes", + "virtiofsd": virtiofsdBody, + "cloud-hypervisor": chBody, + } { + got, err := os.ReadFile(filepath.Join(outDir, name)) + if err != nil { + t.Fatalf("reading %s: %v", name, err) + } + if string(got) != wantBody { + t.Errorf("%s content = %q, want %q", name, got, wantBody) + } + } + + for _, execName := range []string{"virtiofsd", "cloud-hypervisor"} { + fi, err := os.Stat(filepath.Join(outDir, execName)) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm()&0o111 == 0 { + t.Errorf("%s mode = %v, want executable bit set", execName, fi.Mode()) + } + } + + stamp, err := os.ReadFile(filepath.Join(outDir, assetStampFile)) + if err != nil { + t.Fatalf("reading %s: %v", assetStampFile, err) + } + if want := microvmAssetStamp("amd64", "4.3.1", "v53.0"); string(stamp) != want { + t.Errorf("stamp = %q, want %q", stamp, want) + } +} + +func TestAssembleMicroVMAssetsClearsStampOnFailure(t *testing.T) { + silenceStepLog(t) + + serveFakeReleases(t, validKataArchive(t, "fake-virtiofsd-binary"), "", true) + + outDir := t.TempDir() + stampPath := filepath.Join(outDir, assetStampFile) + if err := os.WriteFile(stampPath, []byte("pre-existing-stamp\n"), 0o644); err != nil { + t.Fatal(err) + } + + if err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.3.1", "v53.0", testAssetDigests()); err == nil { + t.Fatal("assembleMicroVMAssets() succeeded when cloud-hypervisor download aborted mid-stream") + } + if _, err := os.Stat(stampPath); !errors.Is(err, os.ErrNotExist) { + t.Errorf("stamp file after failed assemble: err = %v, want ErrNotExist", err) + } + if _, err := os.Stat(filepath.Join(outDir, "cloud-hypervisor")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("partial cloud-hypervisor after aborted download: err = %v, want ErrNotExist", err) + } +} + +func TestAssembleMicroVMAssetsRejectsDigestMismatchBeforeStamping(t *testing.T) { + silenceStepLog(t) + + serveFakeReleases(t, validKataArchive(t, "bad-virtiofsd-bytes"), "ch", false) + + outDir := t.TempDir() + err := assembleMicroVMAssets(t.Context(), outDir, "amd64", "4.3.1", "v53.0", testAssetDigests()) + if err == nil || !strings.Contains(err.Error(), "sha256 mismatch") { + t.Fatalf("assembleMicroVMAssets() = %v, want sha256 mismatch error", err) + } + if _, statErr := os.Stat(filepath.Join(outDir, assetStampFile)); !errors.Is(statErr, os.ErrNotExist) { + t.Errorf("stamp written despite digest mismatch: %v", statErr) + } +} + +func TestExtractKataTarZstRejectsTraversal(t *testing.T) { + for _, tc := range []struct { + name string + entry tarTestEntry + }{ + { + name: "dotdot path", + entry: tarTestEntry{ + name: "opt/kata/share/kata-containers/../../../../../etc/passwd", + typeflag: tar.TypeReg, + body: "bad", + }, + }, + { + name: "symlink escaping workDir", + entry: tarTestEntry{ + name: "opt/kata/share/kata-containers/vmlinux.container", + typeflag: tar.TypeSymlink, + linkname: "../../../../../../etc/passwd", + }, + }, + { + name: "absolute symlink target", + entry: tarTestEntry{ + name: "opt/kata/share/kata-containers/vmlinux.container", + typeflag: tar.TypeSymlink, + linkname: "/etc/passwd", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + data := buildKataTarZst(t, []tarTestEntry{tc.entry}) + _, err := extractKataTarZst(t.Context(), bytes.NewReader(data), t.TempDir()) + if err == nil { + t.Fatal("extractKataTarZst() succeeded on escaping entry, want error") + } + }) + } +} + +func TestMicroVMSandboxConfigTemplate(t *testing.T) { + tmplPath := filepath.Join(repoRoot(t), microvmSandboxConfigTemplate) + digests, err := loadMicroVMAssetDigests(tmplPath) + if err != nil { + t.Fatalf("loadMicroVMAssetDigests(%s) = %v", tmplPath, err) + } + for _, arch := range microvmArches { + for _, asset := range microvmAssets { + d := digests[arch][asset] + if len(d) != 64 { + t.Errorf("digests[%s][%s] = %q, want 64-char sha256 hex", arch, asset, d) + } + } + } + for _, asset := range microvmAssets { + if digests["amd64"][asset] == digests["arm64"][asset] { + t.Errorf("amd64 and arm64 share the same digest for %s: %s", asset, digests["amd64"][asset]) + } + } + + for _, tc := range []struct { + scheme string + bucket string + }{ + {scheme: "gs", bucket: "my-test-bucket"}, + {scheme: "s3", bucket: defaultKindMicroVMBucket}, + } { + t.Run(tc.scheme, func(t *testing.T) { + rendered, err := render.Template(tmplPath, map[string]string{ + "BUCKET_SCHEME": tc.scheme, + "ASSET_BUCKET_NAME": tc.bucket, + }, nil) + if err != nil { + t.Fatalf("render.Template(%s) = %v", tmplPath, err) + } + if strings.Contains(string(rendered), "${") { + t.Errorf("rendered manifest still contains unexpanded placeholder:\n%s", rendered) + } + for _, arch := range microvmArches { + for _, asset := range microvmAssets { + wantDigest := digests[arch][asset] + wantURL := tc.scheme + "://" + tc.bucket + "/" + microvmAssetsPrefix + "/" + arch + "/" + wantDigest + if !strings.Contains(string(rendered), wantURL) { + t.Errorf("rendered manifest missing expected bucket URL %q:\n%s", wantURL, rendered) + } + } + } + objs, err := kube.DecodeManifestBytes(rendered) + if err != nil { + t.Fatalf("DecodeManifestBytes() = %v", err) + } + wantGVK := atev1.GroupVersion.WithKind("SandboxConfig") + if len(objs) != 1 || objs[0].GroupVersionKind() != wantGVK || objs[0].GetName() != microvmSandboxConfigName { + t.Errorf("decoded objects = %v, want single %v/%s", objs, wantGVK, microvmSandboxConfigName) + } + }) + } +} + +func TestMicroVMArches(t *testing.T) { + kindEnv := &Env{Cfg: &config.Config{Kind: true}} + if got, want := kindEnv.microVMArches(), []string{runtime.GOARCH}; !reflect.DeepEqual(got, want) { + t.Errorf("Kind microVMArches() = %v, want %v", got, want) + } + gkeEnv := &Env{Cfg: &config.Config{Kind: false}} + if got, want := gkeEnv.microVMArches(), []string{"amd64", "arm64"}; !reflect.DeepEqual(got, want) { + t.Errorf("non-Kind microVMArches() = %v, want %v", got, want) + } +} + +func TestParseSandboxClasses(t *testing.T) { + for _, tc := range []struct { + raw string + wantGvisor bool + wantMicroVM bool + wantErr bool + }{ + {raw: "", wantGvisor: true, wantMicroVM: true}, + {raw: "gvisor,microvm", wantGvisor: true, wantMicroVM: true}, + {raw: "microvm,gvisor", wantGvisor: true, wantMicroVM: true}, + {raw: "gvisor", wantGvisor: true, wantMicroVM: false}, + {raw: "microvm", wantGvisor: false, wantMicroVM: true}, + {raw: "kata", wantErr: true}, + {raw: "gvisor,", wantErr: true}, + } { + t.Run(tc.raw, func(t *testing.T) { + gvisor, microvm, err := parseSandboxClasses(tc.raw) + if (err != nil) != tc.wantErr { + t.Fatalf("parseSandboxClasses(%q) err = %v, wantErr %v", tc.raw, err, tc.wantErr) + } + if gvisor != tc.wantGvisor || microvm != tc.wantMicroVM { + t.Errorf("parseSandboxClasses(%q) = (%v, %v), want (%v, %v)", tc.raw, gvisor, microvm, tc.wantGvisor, tc.wantMicroVM) + } + }) + } +} + +func TestStageMicroVMAssetsToGCS(t *testing.T) { + silenceStepLog(t) + + tmplPath := filepath.Join(repoRoot(t), microvmSandboxConfigTemplate) + digests, err := loadMicroVMAssetDigests(tmplPath) + if err != nil { + t.Fatal(err) + } + + root := t.TempDir() + binDir := t.TempDir() + callsFile := filepath.Join(t.TempDir(), "gcloud-calls.txt") + existingObj := "gs://test-bucket-assets/" + microvmAssetsPrefix + "/amd64/" + digests["amd64"]["vmlinux"] + script := "#!/bin/sh\necho \"$*\" >> " + callsFile + "\nif [ \"$2\" = \"ls\" ]; then\n echo \"" + existingObj + "\"\nfi\n" + if err := os.WriteFile(filepath.Join(binDir, "gcloud"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + for _, arch := range microvmArches { + dir := filepath.Join(root, "bin", microvmAssetsPrefix, arch) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for _, f := range microvmAssets { + if err := os.WriteFile(filepath.Join(dir, f), []byte(f), 0o644); err != nil { + t.Fatal(err) + } + } + } + + env := &Env{Cfg: &config.Config{Root: root, BucketName: "test-bucket"}} + if err := env.stageMicroVMAssetsToGCS(t.Context(), "test-bucket-assets", digests); err != nil { + t.Fatalf("stageMicroVMAssetsToGCS() = %v", err) + } + + gotBytes, err := os.ReadFile(callsFile) + if err != nil { + t.Fatal(err) + } + got := string(gotBytes) + for _, arch := range microvmArches { + for _, f := range microvmAssets { + src := filepath.Join(root, "bin", microvmAssetsPrefix, arch, f) + dst := "gs://test-bucket-assets/" + microvmAssetsPrefix + "/" + arch + "/" + digests[arch][f] + want := "storage cp " + src + " " + dst + if dst == existingObj { + if strings.Contains(got, want) { + t.Errorf("expected existing object %q to be skipped, but found %q in:\n%s", dst, want, got) + } + continue + } + if !strings.Contains(got, want) { + t.Errorf("missing expected gcloud upload %q in:\n%s", want, got) + } + } + } +} + +func TestStageMicroVMAssetsToRustfs(t *testing.T) { + silenceStepLog(t) + + tmplPath := filepath.Join(repoRoot(t), microvmSandboxConfigTemplate) + digests, err := loadMicroVMAssetDigests(tmplPath) + if err != nil { + t.Fatal(err) + } + + root := t.TempDir() + binDir := t.TempDir() + callsFile := filepath.Join(t.TempDir(), "docker-calls.txt") + existingKey := microvmAssetsPrefix + "/" + runtime.GOARCH + "/" + digests[runtime.GOARCH]["vmlinux"] + script := "#!/bin/sh\necho \"$*\" >> " + callsFile + "\ncase \"$*\" in\n *\"s3 ls --recursive\"*)\n echo \"2026-10-28 00:00:00 123 " + existingKey + "\"\n ;;\nesac\n" + if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + archDir := filepath.Join(root, "bin", microvmAssetsPrefix, runtime.GOARCH) + if err := os.MkdirAll(archDir, 0o755); err != nil { + t.Fatal(err) + } + for _, f := range microvmAssets { + if err := os.WriteFile(filepath.Join(archDir, f), []byte(f), 0o644); err != nil { + t.Fatal(err) + } + } + + replicas := int32(1) + cs := fake.NewClientset( + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "kind-worker"}}, + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "kind-worker2"}}, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "rustfs", Namespace: "ate-system", Generation: 1}, + Spec: appsv1.DeploymentSpec{Replicas: &replicas}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, + Replicas: 1, + UpdatedReplicas: 1, + ReadyReplicas: 1, + AvailableReplicas: 1, + }, + }, + &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "rustfs-bucket-init", Namespace: "ate-system"}, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{{ + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }}, + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rustfs-old-failed", + Namespace: "ate-system", + Labels: map[string]string{"app": "rustfs"}, + }, + Spec: corev1.PodSpec{NodeName: "kind-worker"}, + Status: corev1.PodStatus{Phase: corev1.PodFailed}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rustfs-abc", + Namespace: "ate-system", + Labels: map[string]string{"app": "rustfs"}, + }, + Spec: corev1.PodSpec{NodeName: "kind-worker2"}, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "rustfs", Namespace: "ate-system"}, + Spec: corev1.ServiceSpec{ClusterIP: "fd00:10:96::1234"}, + }, + ) + + env := &Env{ + Cfg: &config.Config{Root: root, Kind: true}, + Kube: kube.NewFromClients(nil, cs, nil, cs.Discovery()), + } + if err := env.stageMicroVMAssetsToRustfs(t.Context(), defaultKindMicroVMBucket, digests); err != nil { + t.Fatalf("stageMicroVMAssetsToRustfs() = %v", err) + } + + gotBytes, err := os.ReadFile(callsFile) + if err != nil { + t.Fatal(err) + } + got := string(gotBytes) + if !strings.Contains(got, "--network container:kind-worker2") { + t.Errorf("expected rustfs pod's node (kind-worker2) in docker args, got:\n%s", got) + } + if !strings.Contains(got, "AWS_ENDPOINT_URL=http://[fd00:10:96::1234]:9000") { + t.Errorf("expected bracketed IPv6 endpoint URL in docker args, got:\n%s", got) + } + for _, f := range microvmAssets { + key := microvmAssetsPrefix + "/" + runtime.GOARCH + "/" + digests[runtime.GOARCH][f] + want := "s3 cp - s3://" + defaultKindMicroVMBucket + "/" + key + if key == existingKey { + if strings.Contains(got, want) { + t.Errorf("expected existing object %q to be skipped, but found %q in:\n%s", key, want, got) + } + continue + } + if !strings.Contains(got, want) { + t.Errorf("missing expected rustfs upload %q in:\n%s", want, got) + } + } +} + +func TestPrepareMicroVMDepsRequiresBucketOnNonKind(t *testing.T) { + silenceStepLog(t) + + cs := fake.NewClientset() + env := &Env{ + Cfg: &config.Config{Root: t.TempDir(), Kind: false}, + Kube: kube.NewFromClients(nil, cs, nil, cs.Discovery()), + } + if _, err := env.prepareMicroVMDeps(t.Context()); err == nil || !strings.Contains(err.Error(), "BUCKET_NAME") { + t.Fatalf("prepareMicroVMDeps() on non-Kind without BucketName = %v, want BUCKET_NAME error", err) + } + err := env.DeploySandboxConfig(t.Context(), DefaultSandboxClasses) + if err == nil || !strings.Contains(err.Error(), "BUCKET_NAME") { + t.Fatalf("DeploySandboxConfig() on non-Kind without BucketName = %v, want BUCKET_NAME error", err) + } + if got := len(cs.Actions()); got != 0 { + t.Errorf("DeploySandboxConfig() made %d API calls before preflight failed, want 0", got) + } +} + +func TestCheckMicroVMDepsRequiresStageTool(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + + t.Run("Kind requires docker", func(t *testing.T) { + env := &Env{Cfg: &config.Config{Kind: true}} + if _, err := env.checkMicroVMDeps(); err == nil || !strings.Contains(err.Error(), "docker") { + t.Fatalf("checkMicroVMDeps() on Kind without docker = %v, want docker error", err) + } + }) + + t.Run("Non-Kind requires gcloud", func(t *testing.T) { + env := &Env{Cfg: &config.Config{Kind: false, BucketName: "test-bucket"}} + if _, err := env.checkMicroVMDeps(); err == nil || !strings.Contains(err.Error(), "gcloud") { + t.Fatalf("checkMicroVMDeps() on non-Kind without gcloud = %v, want gcloud error", err) + } + }) +} + +func TestMicroVMBucket(t *testing.T) { + for _, tc := range []struct { + name string + cfg config.Config + wantBucket string + wantScheme string + wantErr bool + }{ + { + name: "Kind defaults to ate-snapshots over s3", + cfg: config.Config{Kind: true}, + wantBucket: defaultKindMicroVMBucket, + wantScheme: "s3", + }, + { + name: "Kind honors explicit BucketName", + cfg: config.Config{Kind: true, BucketName: "custom-kind-bucket"}, + wantBucket: "custom-kind-bucket", + wantScheme: "s3", + }, + { + name: "Non-Kind requires explicit BucketName", + cfg: config.Config{Kind: false}, + wantScheme: "gs", + wantErr: true, + }, + { + name: "Non-Kind uses -assets over gs", + cfg: config.Config{Kind: false, BucketName: "my-gke-bucket"}, + wantBucket: "my-gke-bucket-assets", + wantScheme: "gs", + }, + { + name: "Non-Kind accepts 56-character BucketName (63-character asset bucket)", + cfg: config.Config{Kind: false, BucketName: strings.Repeat("a", 56)}, + wantBucket: strings.Repeat("a", 56) + "-assets", + wantScheme: "gs", + }, + { + name: "Non-Kind rejects BucketName whose asset bucket exceeds 63 characters", + cfg: config.Config{Kind: false, BucketName: strings.Repeat("a", 57)}, + wantScheme: "gs", + wantErr: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + env := &Env{Cfg: &tc.cfg} + gotBucket, err := env.microVMBucket() + if (err != nil) != tc.wantErr { + t.Fatalf("microVMBucket() error = %v, wantErr %v", err, tc.wantErr) + } + if gotBucket != tc.wantBucket { + t.Errorf("microVMBucket() = %q, want %q", gotBucket, tc.wantBucket) + } + if gotScheme := env.microVMBucketScheme(); gotScheme != tc.wantScheme { + t.Errorf("microVMBucketScheme() = %q, want %q", gotScheme, tc.wantScheme) + } + }) + } +} + +func TestMicroVMDepsStepsAreSeparableFromContext(t *testing.T) { + silenceStepLog(t) + + env := &Env{Cfg: &config.Config{Root: t.TempDir(), Kind: true}} + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + if _, err := env.prepareMicroVMDeps(ctx); err == nil { + t.Error("prepareMicroVMDeps() with a cancelled context = nil, want an error") + } + if err := env.applyMicroVMSandboxConfig(ctx, µVMPrepared{}); err == nil { + t.Error("applyMicroVMSandboxConfig() with a cancelled context = nil, want an error") + } +} diff --git a/cmd/ateworker-microvm/internal/third_party/kata/PROVENANCE.md b/cmd/ateworker-microvm/internal/third_party/kata/PROVENANCE.md index d75a1d5d1e..23a3cc902f 100644 --- a/cmd/ateworker-microvm/internal/third_party/kata/PROVENANCE.md +++ b/cmd/ateworker-microvm/internal/third_party/kata/PROVENANCE.md @@ -6,7 +6,7 @@ everything under it. - **Upstream:** github.com/kata-containers/kata-containers - **Version:** tag `3.31.0`. The microVM demo's runtime assets track a newer kata - (see `hack/microvm-assets/assemble.sh` `KATA_VER`, currently `4.3.1`); the four + (see `cmd/ate-setup/internal/steps/microvm.go` `defaultKataVersion`, currently `4.3.1`); the four vendored `.proto` files are byte-identical between `3.31.0` and `4.3.1` (verified 2026-10-23), so the generated Go needs no re-vendor. - **License:** Apache-2.0 — `./LICENSE` is the upstream license verbatim (mirrored to diff --git a/demos/counter/README.md b/demos/counter/README.md index 98bb4b5329..81f7d5a319 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -48,12 +48,18 @@ Run it and follow the printed next steps: KIND_CLUSTER_NAME= ./hack/run-microvm-demo-kind.sh ``` -On a cluster that already has the microVM deps (`hack/install-microvm-deps.sh`), -deploy directly instead: +To deploy onto an existing cluster where `ate-system` is already installed: ```bash go run ./cmd/ate-setup deploy demo counter-microvm ``` + +Inspect the deployed template with: + +```bash +kubectl ate get actor-templates -a ate-demo-counter-microvm +kubectl ate get actor-template counter-microvm -a ate-demo-counter-microvm -o yaml +```
diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index d609850dbf..10c84f745d 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -18,8 +18,7 @@ # memory snapshot round-tripped. The ActorTemplate is not a CRD: it lives in # counter-microvm-template.yaml.tmpl and is created through the ate API with # `kubectl ate create actor-template`. The cluster-wide `microvm` -# SandboxConfig referenced by the pool is installed by -# hack/install-microvm-deps.sh +# SandboxConfig referenced by the pool is installed by `deploy ate-system`. apiVersion: v1 kind: Namespace diff --git a/demos/egress/egress-microvm.yaml.tmpl b/demos/egress/egress-microvm.yaml.tmpl index d4f589b596..edd1f199d7 100644 --- a/demos/egress/egress-microvm.yaml.tmpl +++ b/demos/egress/egress-microvm.yaml.tmpl @@ -17,8 +17,7 @@ # sandbox classes. The ActorTemplate is a substrate resource, not a CRD: it # lives in egress-microvm-template.yaml.tmpl and is created through the # ate API with `kubectl ate create actor-template`. The cluster-wide `microvm` -# SandboxConfig referenced by the pool is installed by -# hack/install-microvm-deps.sh +# SandboxConfig referenced by the pool is installed by `deploy ate-system`. apiVersion: v1 kind: Namespace diff --git a/docs/api-guide.md b/docs/api-guide.md index ffb19a45ff..0773468226 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -564,7 +564,7 @@ spec: A `microvm` `SandboxConfig` supplies the [Kata Containers](https://katacontainers.io/) + [Cloud Hypervisor](https://www.cloudhypervisor.org/) toolchain instead of `runsc`. Each architecture of each version must define the full asset set — `cloud-hypervisor`, `virtiofsd`, `kata-kernel`, and `kata-image` — which a `ValidatingAdmissionPolicy` enforces at apply time. Worker pods for a microVM pool require `/dev/kvm` and nested-virtualization-capable nodes. The controller requests those devices on the pod automatically, and atelet advertises them only where they exist, so placement follows the hardware rather than a node label. Clusters that reserve nested-virt nodes with an `ate.dev/sandboxClass=microvm` taint are still tolerated: advertising a device attracts these pods to capable nodes but repels nothing else from them. The same convention applies to every class: worker pods of a pool tolerate `ate.dev/sandboxClass=:NoSchedule`, so a cluster can reserve a node pool per sandbox class with that taint, and the atelet DaemonSet tolerates the key for any value. On GKE, a pool can also select a ComputeClass with `template.nodeSelector` (`cloud.google.com/compute-class: `): GKE taints the nodes it creates for the class `cloud.google.com/compute-class=:NoSchedule` and adds the matching toleration to the worker pods, and the atelet DaemonSet tolerates that key as well. -See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. +`go run ./cmd/ate-setup deploy ate-system` (and `deploy sandboxconfig`) assembles and stages these assets automatically; see [`docs/dev/microvm-local.md`](dev/microvm-local.md) for local bring-up and a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. --- diff --git a/docs/dev/microvm-local.md b/docs/dev/microvm-local.md index 66aba29851..4ebcf1aeff 100644 --- a/docs/dev/microvm-local.md +++ b/docs/dev/microvm-local.md @@ -13,8 +13,29 @@ Complete the [Quickstart (Development)](../../README.md#quickstart-development) in the README first — it covers the base tooling and the default (gVisor) path this guide builds on. For background on the runtime, see -[architecture.md](../architecture.md) and -[hack/microvm-assets/README.md](../../hack/microvm-assets/README.md). +[architecture.md](../architecture.md). + +The `microvm` runtime (`cmd/ateworker-microvm`) fetches its guest toolchain at +runtime rather than baking Kata into the worker image. Each actor container's +rootfs is an overlay of a read-only lower (the OCI image, served into the guest +over virtio-fs by `virtiofsd`) and a writable upper on a guest tmpfs, so the +asset set is four files per architecture (`amd64` and `arm64`): + +- `cloud-hypervisor` — the VMM binary (from its release) +- `virtiofsd` — the virtio-fs daemon serving the read-only lower (from `kata-static`) +- `vmlinux` — the guest kernel (from `kata-static`) +- `rootfs.img` — the guest rootfs image (from `kata-static`) + +`go run ./cmd/ate-setup deploy ate-system` (and `deploy sandboxconfig`) +assembles the guest assets into `bin/microvm-assets//` (the host +architecture on kind, and both `amd64` and `arm64` on non-kind), verifies +their SHA-256 digests against `manifests/ate-install/sandboxconfig-microvm.yaml.tmpl`, +stages them into the cluster's object store under `microvm-assets//` +(the shared `rustfs` bucket on kind, and `-assets` on GKE, the only +bucket `setup-gcp` lets nodes read), and applies the cluster-wide `microvm` +`SandboxConfig`. Assets must not be stored under a snapshot location (an +ActorTemplate's `snapshotConfig.location` followed by `/atespaces/`), as the +node's object-store plugin refuses to serve anything there as an asset. ## Option A: Linux host with KVM @@ -56,12 +77,11 @@ kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.c ./hack/run-microvm-demo-kind.sh ``` -This is a one-shot bring-up: it deploys the control plane, installs the -cluster-wide microVM deps via `hack/install-microvm-deps.sh` — assembling the -guest runtime assets for your architecture (skipped if already present under -`bin/microvm-assets/`), staging them into the in-cluster rustfs bucket, and -applying the `microvm` `SandboxConfig` — then deploys the demo worker pool + -template. +This is a one-shot bring-up: it deploys the control plane (which assembles the +guest runtime assets for the host architecture if not already present under +`bin/microvm-assets//`, stages them into the in-cluster rustfs bucket, +and applies the cluster-wide `microvm` `SandboxConfig`), then deploys the demo +worker pool + template. ### 4. Verify @@ -147,23 +167,7 @@ limactl stop docker-nested limactl start docker-nested ``` -### 3. Assemble the arm64 assets inside the Lima VM - -Assemble the arm64 assets in the guest because macOS does not ship `zstd` by default: - -```sh -limactl shell docker-nested - -cd # visible via the writable home mount -./hack/microvm-assets/assemble.sh -exit -``` - -The script downloads about 590 MB and takes a minute or two, leaving ~285 MB in -`bin/microvm-assets/arm64/`. That directory is shared with the host through the -home mount — the demo script will find the assets there and skip re-assembling. - -### 4. Point the Docker CLI at Lima and bring everything up (on macOS) +### 3. Point the Docker CLI at Lima and bring everything up (on macOS) ```sh export DOCKER_HOST="unix://${HOME}/.lima/docker-nested/sock/docker.sock" @@ -175,6 +179,10 @@ cd ./hack/run-microvm-demo-kind.sh ``` +The first run of `run-microvm-demo-kind.sh` downloads the guest runtime +releases and extracts them into `bin/microvm-assets//`; later +runs reuse that directory when the version stamp and SHA-256 digests match. + Verify as in Option A, step 4. ## Trying it out diff --git a/docs/dev/mitm-egress-test-runbook.md b/docs/dev/mitm-egress-test-runbook.md index 2d3a7c978c..21fdf0e4ee 100644 --- a/docs/dev/mitm-egress-test-runbook.md +++ b/docs/dev/mitm-egress-test-runbook.md @@ -25,13 +25,6 @@ kubectl config use-context kind-kind ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' ./hack/install-ate-kind.sh --deploy-ate-system ``` -For microVM, install its assets before deploying the fixture: - -```bash -NO_DEV_ENV=true ATE_INSTALL_KIND=true KUBECTL_CONTEXT=kind-kind \ - ./hack/install-microvm-deps.sh --install -``` - Deploy both egress ActorTemplates and add local-origin trust to the current gateway: diff --git a/docs/operator-install.md b/docs/operator-install.md index 86b46f6d5b..561e57fcba 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -319,6 +319,7 @@ demos, the [benchmarks](../benchmarking/README.md#settings), and the | `publish.overwrite` | `ATE_PUBLISH_OVERWRITE` | `--overwrite` | `false` | `publish release-images` | | `record.dir` | `ATE_RECORD_DIR` | `--record-dir` | — | (global) | | `rolloutTimeout` | `ATE_INSTALL_ROLLOUT_TIMEOUT` | `--rollout-timeout` | `1m0s` | (global) | +| `sandboxClasses` | `ATE_INSTALL_SANDBOX_CLASSES` | `--sandbox-classes` | `gvisor,microvm` | `deploy ate-system`, `deploy sandboxconfig` | | `storage.bucketName` | `BUCKET_NAME` | `--storage-bucket-name` | — | (global) | | `storage.snapshotLocation` | `SNAPSHOT_LOCATION` | `--snapshot-location` | — | (global) | | `storage.transferGSAName` | `SNAPSHOT_TRANSFER_GSA_NAME` | `--snapshot-transfer-gsa-name` | — | (global) | @@ -354,7 +355,7 @@ does not create the bucket or grant access to it: run [tools/setup-gcp](../tools/setup-gcp/README.md)). setup-gcp also creates an asset bucket, `-assets`, where -`hack/install-microvm-deps.sh` stages microVM sandbox assets, and gives atelet +`ate-setup` stages microVM sandbox assets, and gives atelet read access to it. atelet gets no access to the snapshot bucket: every snapshot transfer runs on a token scoped to its actor, issued by ate-api-server's credential exchange. A GCP project set up before the asset bucket existed diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh deleted file mode 100755 index acfe705d23..0000000000 --- a/hack/install-microvm-deps.sh +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Install (or delete) the cluster-wide micro-VM (kata + cloud-hypervisor) -# dependencies. This is opt-in on top of hack/install-ate.sh --deploy-ate-system, -# which only installs the default gVisor SandboxConfig. Used by: -# * hack/run-microvm-demo.sh (before applying the counter-microvm demo) -# * benchmarking/automation/orchestrator.py (before deploying a microvm -# benchmark workload) -# -# On --install: assembles the asset set (assemble.sh; skipped if OUT already -# has them), stages the assets under kata-assets/ to the cluster's object -# store bucket (rustfs on kind, GCS on GKE), and applies the cluster-wide -# `microvm` SandboxConfig referencing those assets. Every asset sha256 is -# pinned in the manifest, so the apply only substitutes the bucket name. -# -# On --delete: removes the SandboxConfig from the cluster. The delete is best -# effort. A SandboxConfig that a WorkerPool uses carries a protection -# finalizer, which atecontroller releases only once no pool's status names -# the config. So the script leaves the config in place if any WorkerPool -# still uses it. Otherwise it waits a bounded time for the finalizer to go. -# Bucket contents are left alone (they're inert until a SandboxConfig points -# at them, and re-staging is cheap on next install). -# -# Like the other hack scripts, this sources .ate-dev-env.sh for the cluster / -# registry / bucket settings unless NO_DEV_ENV is set. -# -# Env (most come from .ate-dev-env.sh): -# BUCKET_NAME object store bucket for snapshots (default: ate-snapshots). On -# kind the assets share it. On GKE they go to BUCKET_NAME-assets, -# which tools/setup-gcp creates, so nodes can read assets -# without being able to read snapshots. -# KUBECTL_CONTEXT (optional) kube context; threaded into kubectl. -# PROJECT_ID (optional) GCP project for the GCS asset upload (GKE path). -# ARCH target arch (default: from KO_DEFAULTPLATFORMS, else host arch). -# OUT asset dir (default: $PWD/bin/microvm-assets/$ARCH, gitignored). -# ATE_INSTALL_KIND "true" for the kind path (stage assets to rustfs); default -# false uploads assets to GCS. - -set -o errexit -o nounset -o pipefail - -ROOT="$(git rev-parse --show-toplevel)" -cd "${ROOT}" - -# Source the environment (cluster, registry, bucket) like the other hack scripts; -# callers set NO_DEV_ENV to skip this and use kind defaults. -if [[ -r .ate-dev-env.sh ]] && [[ -z "${NO_DEV_ENV:-}" ]]; then - source .ate-dev-env.sh -fi - -BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" -KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" -ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" - -usage() { - cat <&2; usage; exit 1 ;; - esac - shift -done - -if [[ -z "${action}" ]]; then - usage - exit 1 -fi - -# kubectl falls back to localhost:8080 when neither --context nor a kubeconfig -# current-context is set, which surfaces mid-install as a confusing "connection -# refused" from the apply -- after the assets have already been assembled and -# staged. Resolve the target cluster up front instead. -if [[ -z "${KUBECTL_CONTEXT}" ]] && ! kubectl config current-context >/dev/null 2>&1; then - echo "Error: no kube context to target: KUBECTL_CONTEXT is empty and the" >&2 - echo " kubeconfig has no current-context." >&2 - echo " Set KUBECTL_CONTEXT (e.g. in .ate-dev-env.sh) or run:" >&2 - echo " kubectl config use-context " >&2 - exit 1 -fi - -# ANSI color codes for prettier output (mirrors hack/install-ate.sh). -COLOR_CYAN='\033[1;36m' -COLOR_RESET='\033[0m' -log() { - echo -e "${COLOR_CYAN}[install-microvm-deps]: $*${COLOR_RESET}" -} - -run_kubectl() { - kubectl ${KUBECTL_CONTEXT:+--context="${KUBECTL_CONTEXT}"} "$@" -} - -MANIFEST_TEMPLATE="manifests/microvm/sandboxconfig-microvm.yaml.tmpl" - -# The asset URLs' scheme: the node's object-store plugin serves only its own -# backend's, which is S3 (rustfs) on kind and GCS on GKE. -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - BUCKET_SCHEME="s3" - ASSET_BUCKET_NAME="${BUCKET_NAME}" -else - BUCKET_SCHEME="gs" - ASSET_BUCKET_NAME="${BUCKET_NAME}-assets" -fi - -if [[ "${action}" == "delete" ]]; then - # A WorkerPool that uses the config holds it. The delete would only mark - # the config Terminating, and `kubectl delete` would wait for it without - # end. A Terminating config also cannot be recreated under its name. So no - # delete is issued at all. The config is left in place and the pools are - # named instead. A cluster without the WorkerPool CRD has no users. - users="$(run_kubectl get workerpools --all-namespaces \ - -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name} {.status.sandboxClasses[*].configRef.name}{"\n"}{end}' 2>/dev/null \ - | awk '{ for (i = 2; i <= NF; i++) if ($i == "microvm") { print $1; break } }' || true)" - if [[ -n "${users}" ]]; then - echo "Warning: the microvm SandboxConfig is left in place; these WorkerPools" >&2 - echo " still use it:" >&2 - while IFS= read -r pool; do echo " ${pool}" >&2; done <<<"${users}" - echo " Delete them, or point their spec.sandboxClasses[].configRef at" >&2 - echo " another microvm SandboxConfig, and run this again." >&2 - exit 0 - fi - log "Deleting microvm SandboxConfig..." - # Delete by name rather than by manifest so a missing/edited template file - # doesn't block cleanup on an older cluster. atecontroller releases the - # protection finalizer within a reconcile. The timeout turns a missing - # atecontroller into an error instead of a hang. - if ! run_kubectl delete --ignore-not-found --timeout=60s sandboxconfig microvm; then - echo "Error: the microvm SandboxConfig did not finish deleting. It stays" >&2 - echo " Terminating until atecontroller releases its protection" >&2 - echo " finalizer; check that atecontroller is running, then run this" >&2 - echo " again." >&2 - exit 1 - fi - log "Done. (Bucket assets at ${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/ left in place.)" - exit 0 -fi - -# --- install ---------------------------------------------------------------- - -# Target arch: match the images' platform (KO_DEFAULTPLATFORMS is set by -# .ate-dev-env.sh on GKE and by the kind wrapper); fall back to the host arch. -if [[ -z "${ARCH:-}" ]]; then - if [[ -n "${KO_DEFAULTPLATFORMS:-}" ]]; then - ARCH="${KO_DEFAULTPLATFORMS##*/}" - else - ARCH="$(go env GOARCH)" - fi -fi -OUT="${OUT:-${ROOT}/bin/microvm-assets/$ARCH}" - -# --- 1. assets: assemble (if missing or stale) ----------------------------- -need_assemble=false -for f in cloud-hypervisor virtiofsd vmlinux rootfs.img; do - if [[ ! -f "${OUT}/${f}" ]]; then - need_assemble=true - break - fi -done -# Presence alone is not enough. The filenames don't change when a version pin -# moves, so an asset dir assembled before a bump looks complete while holding the old -# bytes — we'd then stage those against a SandboxConfig pinning the new shas, and the -# mismatch would only surface at runtime as an actor wedged in STATUS_RESUMING while -# atelet rejects the download. Compare the stamp assemble.sh left against the one it -# would write now; a dir predating the stamp, or left by a failed assemble (which clears -# the stamp before overwriting anything), has no file and re-assembles. -if [[ "${need_assemble}" == "false" ]]; then - want_stamp="$(ARCH="${ARCH}" hack/microvm-assets/assemble.sh --print-stamp)" - if [[ "$(cat "${OUT}/.asset-versions" 2>/dev/null)" != "${want_stamp}" ]]; then - log "Asset set in ${OUT} is stale (version stamp mismatch); re-assembling." - need_assemble=true - fi -fi -if [[ "${need_assemble}" == "true" ]]; then - log "Assembling micro-VM assets into ${OUT} (ARCH=${ARCH})..." - ARCH="${ARCH}" OUT="${OUT}" hack/microvm-assets/assemble.sh -else - log "Assets already present in ${OUT}; skipping assemble." -fi - -# --- 2. stage assets to rustfs (kind) / GCS (GKE) -------------------------- -# Upload the four assets under kata-assets/, where atelet fetches them: the -# in-cluster rustfs (S3 API) on kind, or the GCS asset bucket on GKE. -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - log "Staging assets to in-cluster rustfs bucket ${BUCKET_NAME} (kata-assets/)..." - OUT="${OUT}" BUCKET="${BUCKET_NAME}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/microvm-assets/stage-to-rustfs.sh -else - log "Uploading assets to gs://${ASSET_BUCKET_NAME}/kata-assets/ ..." - OUT="${OUT}" BUCKET="${ASSET_BUCKET_NAME}" hack/microvm-assets/stage-to-gcs.sh -fi - -# --- 3. apply the cluster-wide microvm SandboxConfig ----------------------- -# Every asset is downloaded rather than built, so all four carry committed, -# reproducible per-arch shas and the bucket's scheme and name are the only -# substitutions left. -log "Applying microvm SandboxConfig from ${MANIFEST_TEMPLATE}..." -sed -e "s|\${BUCKET_SCHEME}|${BUCKET_SCHEME}|g" \ - -e "s|\${ASSET_BUCKET_NAME}|${ASSET_BUCKET_NAME}|g" \ - "${MANIFEST_TEMPLATE}" \ - | run_kubectl apply -f - - -log "Done. Installed cluster-wide microvm SandboxConfig." diff --git a/hack/microvm-assets/README.md b/hack/microvm-assets/README.md deleted file mode 100644 index 1436ef4e68..0000000000 --- a/hack/microvm-assets/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# MicroVM runtime assets + counter demo (kind, fetch-not-bake) - -The `microvm` runtime (`cmd/ateworker-microvm`, kata + cloud-hypervisor) fetches its -toolchain at runtime — nothing kata-specific is baked into the worker image. ateworker drives -the kata-agent directly (no kata shim, no containerd). Each actor container's rootfs is an -overlay of a read-only lower (the OCI image, served into the guest over virtio-fs by -`virtiofsd`) and a writable upper on a guest tmpfs, so `virtiofsd` is part of the asset -set. The asset set is four files: - -- `cloud-hypervisor` — the VMM binary (fetched from its release) -- `virtiofsd` — the virtio-fs daemon serving the RO lower (from kata-static) -- `vmlinux` — the guest kernel (from kata-static) -- `rootfs.img` — the guest rootfs image (from kata-static) - -These helpers assemble the asset set for your node arch, stage it into the cluster's rustfs -S3 bucket, and the demo manifest's `SandboxConfig` points at it. When `/dev/kvm` is -available, `hack/create-kind-cluster.sh` mounts it into the node; atelet then advertises -it as a device, which is what places microVM workers there. - -> [!TIP] -> `hack/run-microvm-demo.sh` automates the full bring-up below (assets, control plane, -> demo apply) for kind OR GKE without editing committed files. The steps here are the -> manual equivalent. - -## Steps (run on a KVM-capable Linux host matching the node arch) - -1. **Assemble assets for your arch:** - ```sh - ARCH=arm64 hack/microvm-assets/assemble.sh - ``` - -2. **Bring up the cluster + control plane:** - ```sh - hack/create-kind-cluster.sh # mounts /dev/kvm into the nodes - hack/install-ate-kind.sh # control plane + rustfs (bucket: ate-snapshots) - ``` - -3. **Stage assets into rustfs:** - ```sh - OUT="$PWD/microvm-assets-arm64" hack/microvm-assets/stage-to-rustfs.sh - ``` - -4. **Apply the demo + drive it:** - ```sh - BUCKET_NAME=ate-snapshots envsubst < demos/counter/counter-microvm.yaml.tmpl | kubectl apply -f - - ``` - Create an actor from `counter-microvm`, hit the in-RAM counter to increment it, suspend - (checkpoint), resume on a different worker pod, and confirm the count continues — proving the - guest-memory snapshot round-tripped across pods. - -## Notes -- Assets must not be stored under a snapshot location (an ActorTemplate's - `snapshotConfig.location` followed by `/atespaces/`): the node's object-store plugin - refuses to serve anything there as an asset. The staging scripts use `kata-assets/`, - which is outside it. On GKE they stage into the asset bucket, `BUCKET_NAME-assets`, - the only bucket setup-gcp lets nodes read. On kind they share the rustfs bucket. -- `assets` is single-arch (unlike runsc's amd64/arm64): stage assets matching the node arch. diff --git a/hack/microvm-assets/assemble.sh b/hack/microvm-assets/assemble.sh deleted file mode 100755 index fc1e09f095..0000000000 --- a/hack/microvm-assets/assemble.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Assemble the micro-VM (kata + cloud-hypervisor) runtime asset set that -# ateworker-microvm fetches at runtime (fetch-not-bake). Run this on a Linux -# host of the TARGET arch. -# -# Produces, under $OUT, the four assets named as the SandboxConfig expects: -# cloud-hypervisor virtiofsd vmlinux rootfs.img -# Every asset is downloaded rather than built, so all four have reproducible bytes: -# paste their sha256 sums into the manifest -# (manifests/microvm/sandboxconfig-microvm.yaml.tmpl). -# -# ateworker drives the kata-agent directly (the kata containerd shim is NOT an asset). The -# actor rootfs is overlay(virtio-fs RO lower + guest-tmpfs upper), so virtiofsd IS an -# asset. CH's restore handshake hangs against virtiofsd v1.13.3, which kata bundled up -# to and including 4.0.0; kata 4.1.0 and later bundle v1.14.0, the first release carrying the -# vhost-0.16 / vhost-user-backend-0.22 snapshot-restore fix (REPLY_ACK). So virtiofsd -# now comes out of kata-static with the kernel and rootfs instead of being sourced -# separately per arch. -# -# Env: ARCH (arm64|amd64, default arm64), KATA_VER (4.3.1), CH_VER (v53.0), -# OUT (default ./bin/microvm-assets/$ARCH, under the gitignored bin/). -# -# Always re-downloads and overwrites — there is no incremental mode. It clears -# $OUT/.asset-versions before the first write and re-stamps it with the versions that -# produced the set only once the run completes, so the stamp is present only on a dir -# assembled end-to-end by those pins. install-microvm-deps.sh uses it to decide whether -# a cached $OUT is still current. -# `--print-stamp` prints that stamp for the current env and exits without downloading. - -set -o errexit -o nounset -o pipefail - -ROOT="$(git rev-parse --show-toplevel)" - -ARCH="${ARCH:-arm64}" -KATA_VER="${KATA_VER:-4.3.1}" -CH_VER="${CH_VER:-v53.0}" -# Not env-overridable: this is whatever KATA_VER bundles. It is declared rather than -# read off the binary because --print-stamp has to answer before anything is -# downloaded, and checked against the extracted binary below so it cannot drift from -# what kata ships. -VIRTIOFSD_VER="1.14.0" -OUT="${OUT:-${ROOT}/bin/microvm-assets/$ARCH}" - -case "$ARCH" in - arm64) CH_ASSET="cloud-hypervisor-static-aarch64" ;; - amd64) CH_ASSET="cloud-hypervisor-static" ;; - *) echo "unsupported ARCH=$ARCH" >&2; exit 1 ;; -esac - -# Identifies the asset set this script produces. Cleared before the first write into -# $OUT and re-written to $OUT/$STAMP_FILE on success; install-microvm-deps.sh compares -# it against what the current checkout would build, because the filenames stay the -# same when a pin moves and an asset dir from an older checkout is otherwise -# indistinguishable from a current one. virtiofsd is stamped even though KATA_VER -# already determines it: its version is what the CH restore handshake turns on, so the -# dir should say which one it holds. -STAMP_FILE=".asset-versions" -asset_stamp() { - printf 'arch=%s\nkata=%s\ncloud-hypervisor=%s\nvirtiofsd=%s\n' \ - "$ARCH" "$KATA_VER" "$CH_VER" "$VIRTIOFSD_VER" -} - -if [ "${1:-}" = "--print-stamp" ]; then - asset_stamp - exit 0 -fi - -WORK="$(mktemp -d)" -trap 'rm -rf "$WORK"' EXIT - -mkdir -p "$OUT" -# Drop any stamp before the first overwrite into $OUT. Assets are replaced in place, -# so a run that dies partway leaves a dir mixing old and new bytes; the stamp it -# inherited describes neither. Clearing it up front means an unstamped dir is the only -# thing a failed run can leave, whatever the pins were before. -rm -f "${OUT}/${STAMP_FILE}" -cd "$WORK" - -echo ">> Downloading kata-static ${KATA_VER} (${ARCH})..." -curl -fSL -o kata-static.tar.zst \ - "https://github.com/kata-containers/kata-containers/releases/download/${KATA_VER}/kata-static-${KATA_VER}-${ARCH}.tar.zst" -mkdir -p kata -tar --zstd -xf kata-static.tar.zst -C kata -KROOT="kata/opt/kata" - -cp "$(readlink -f "${KROOT}/share/kata-containers/vmlinux.container")" "${OUT}/vmlinux" -# rootfs.img is attached over virtio-pmem and mounted with DAX (cmd/ateworker-microvm/run.go). -# The image must meet the following requirements: -# - a size that is a multiple of 2 MiB (cloud-hypervisor) -# - partition 1 aligned to the guest page size -# - a filesystem block size equal to the guest page size -# The kata-containers.img used here is built for this, and vmlinux.container uses 4 KiB pages. -# See https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder/image-builder/image_builder.sh -cp "$(readlink -f "${KROOT}/share/kata-containers/kata-containers.img")" "${OUT}/rootfs.img" -# Statically linked, so it runs as-is outside the kata layout it is packaged for. -cp "${KROOT}/libexec/virtiofsd" "${OUT}/virtiofsd" -chmod +x "${OUT}/virtiofsd" - -echo ">> Downloading cloud-hypervisor ${CH_VER} (${CH_ASSET})..." -curl -fSL -o "${OUT}/cloud-hypervisor" \ - "https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${CH_VER}/${CH_ASSET}" -chmod +x "${OUT}/cloud-hypervisor" - -echo -echo ">> Assets assembled in ${OUT}:" -cd "${OUT}" -for f in cloud-hypervisor virtiofsd vmlinux rootfs.img; do - [ -f "$f" ] || { echo "MISSING: $f" >&2; exit 1; } -done -# The stamp names a virtiofsd version, so confirm the tarball carried that one before -# writing it: a kata-side bump would otherwise stamp a version this dir does not hold. -# Only checkable where the binary runs, and assembling for another arch (or on macOS) -# is legitimate, so a binary this host cannot exec is skipped rather than fatal. -if GOT_VIRTIOFSD="$("${OUT}/virtiofsd" --version 2>/dev/null | head -1 | awk '{print $2}')" \ - && [ -n "${GOT_VIRTIOFSD}" ]; then - if [ "${GOT_VIRTIOFSD}" != "${VIRTIOFSD_VER}" ]; then - echo "kata ${KATA_VER} bundles virtiofsd ${GOT_VIRTIOFSD}, not ${VIRTIOFSD_VER}: update VIRTIOFSD_VER" >&2 - exit 1 - fi - echo "virtiofsd ${GOT_VIRTIOFSD}" -fi -# Written only once all four are present and virtiofsd matches, and only after the -# up-front rm, so the stamp exists exactly when this dir was assembled end-to-end by -# these pins. -asset_stamp > "${OUT}/${STAMP_FILE}" -echo -echo ">> sha256 (paste all four into the per-arch block in" -echo ">> manifests/microvm/sandboxconfig-microvm.yaml.tmpl):" -sha256sum cloud-hypervisor virtiofsd vmlinux rootfs.img diff --git a/hack/microvm-assets/stage-to-gcs.sh b/hack/microvm-assets/stage-to-gcs.sh deleted file mode 100755 index 7820e6c9c3..0000000000 --- a/hack/microvm-assets/stage-to-gcs.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Stage the assembled micro-VM asset set into the GCS asset bucket under -# kata-assets/, where atelet fetches it through the node's object-store plugin -# (per manifests/microvm/sandboxconfig-microvm.yaml.tmpl). The plugin refuses -# assets stored under a snapshot location (/atespaces/), -# so keep them outside it. The GKE counterpart of stage-to-rustfs.sh. Run after -# assemble.sh has produced $OUT. -# -# Requires the `gcloud` CLI authenticated for the bucket's project. Env: OUT (asset -# dir, default ./bin/microvm-assets/amd64), BUCKET (required, the asset bucket, -# BUCKET_NAME-assets as install-microvm-deps.sh passes it), PROJECT_ID (optional; -# passed to gcloud as --project when set). - -set -o errexit -o nounset -o pipefail - -ROOT="$(git rev-parse --show-toplevel)" - -OUT="${OUT:-${ROOT}/bin/microvm-assets/amd64}" -BUCKET="${BUCKET:?set BUCKET to the asset bucket, BUCKET_NAME-assets}" - -# Pass --project only when PROJECT_ID is set (mirrors hack/teardown.sh); otherwise -# gcloud uses its active config project. ${PROJECT_ID:+...} elides the flag entirely -# when unset (same idiom as KUBECTL_CONTEXT in hack/run-microvm-demo.sh). -echo ">> Uploading assets to gs://${BUCKET}/kata-assets/ ..." -for f in cloud-hypervisor virtiofsd vmlinux rootfs.img; do - echo " $f" - gcloud storage cp ${PROJECT_ID:+--project="${PROJECT_ID}"} "${OUT}/${f}" "gs://${BUCKET}/kata-assets/${f}" -done - -echo ">> Done. Verify:" -gcloud storage ls ${PROJECT_ID:+--project="${PROJECT_ID}"} "gs://${BUCKET}/kata-assets/" diff --git a/hack/microvm-assets/stage-to-rustfs.sh b/hack/microvm-assets/stage-to-rustfs.sh deleted file mode 100755 index 5fbd9c1e8b..0000000000 --- a/hack/microvm-assets/stage-to-rustfs.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Stage the assembled micro-VM asset set into the kind cluster's rustfs S3 bucket -# under kata-assets/, where atelet fetches it through the node's object-store -# plugin (per manifests/microvm/sandboxconfig-microvm.yaml.tmpl). The plugin -# refuses assets stored under a snapshot location -# (/atespaces/), so keep them outside it. -# Run after the cluster is up (hack/install-ate-kind.sh) and assemble.sh has produced $OUT. -# -# The S3 client runs in a throwaway container (the same pinned amazon/aws-cli -# image as the rustfs-bucket-init Job), so developers don't need the `aws` CLI -# installed. The container joins the kind node's network namespace, which is what -# makes rustfs's ClusterIP routable: a container merely attached to the `kind` -# docker network can reach the node's own IP but has no route to the service or -# pod CIDRs. That also avoids a `kubectl port-forward`, which silently uploads -# nothing when it targets the wrong cluster, and which a container can't reach at -# all on macOS (the container's localhost is the Docker VM, not the host). -# -# Env: OUT (asset dir, default ./bin/microvm-assets/arm64), BUCKET (default -# ate-snapshots), NAMESPACE (rustfs namespace, default ate-system), -# KUBECTL_CONTEXT (optional; kube context), KIND_CLUSTER_NAME (default: derived -# from KUBECTL_CONTEXT, else "kind"). - -set -o errexit -o nounset -o pipefail - -ROOT="$(git rev-parse --show-toplevel)" - -OUT="${OUT:-${ROOT}/bin/microvm-assets/arm64}" -BUCKET="${BUCKET:-ate-snapshots}" -NAMESPACE="${NAMESPACE:-ate-system}" -KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" -# kind contexts are named kind-; fall back to kind's own default. -KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-${KUBECTL_CONTEXT#kind-}}" -KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" - -# Keep in sync with the rustfs-bucket-init Job in -# manifests/ate-install/kind/rustfs.yaml, which creates the bucket we upload into. -AWS_CLI_IMAGE="amazon/aws-cli:2.38.5@sha256:c8156a0e74e13929a42d7e60f0d4e14e7d28d980cd21822505d4bff80e5460c1" - -ASSETS=(cloud-hypervisor virtiofsd vmlinux rootfs.img) - -run_kubectl() { - kubectl ${KUBECTL_CONTEXT:+--context="${KUBECTL_CONTEXT}"} -n "${NAMESPACE}" "$@" -} - -if ! command -v docker >/dev/null 2>&1; then - echo "error: 'docker' is required to run the S3 client but was not found in PATH" >&2 - exit 1 -fi - -for f in "${ASSETS[@]}"; do - if [[ ! -f "${OUT}/${f}" ]]; then - echo "error: missing asset ${OUT}/${f}; run hack/microvm-assets/assemble.sh first" >&2 - exit 1 - fi -done - -# rustfs must be serving and the bucket must exist before anything is uploaded. -echo ">> Waiting for rustfs in namespace ${NAMESPACE}..." -run_kubectl rollout status deploy/rustfs --timeout=300s -run_kubectl wait --for=condition=Complete job/rustfs-bucket-init --timeout=300s - -NODE="$("${ROOT}/hack/kind.sh" get nodes --name "${KIND_CLUSTER_NAME}" | head -n1)" -if [[ -z "${NODE}" ]]; then - echo "error: no nodes found for kind cluster '${KIND_CLUSTER_NAME}'" >&2 - exit 1 -fi -ENDPOINT="http://$(run_kubectl get svc rustfs -o jsonpath='{.spec.clusterIP}'):9000" - -echo ">> Uploading assets to s3://${BUCKET}/kata-assets/ via ${ENDPOINT} (netns of ${NODE})..." -aws_cli() { - # -i (no -t): a TTY would corrupt the binary stream piped in on stdin. - docker run --rm -i \ - --network "container:${NODE}" \ - -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-rustfsadmin}" \ - -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-rustfsadmin}" \ - -e AWS_REGION="${AWS_REGION:-us-east-1}" \ - -e AWS_ENDPOINT_URL="${ENDPOINT}" \ - "${AWS_CLI_IMAGE}" "$@" -} - -for f in "${ASSETS[@]}"; do - echo " $f" - aws_cli s3 cp - "s3://${BUCKET}/kata-assets/${f}" < "${OUT}/${f}" -done - -echo ">> Done. Verify:" -aws_cli s3 ls "s3://${BUCKET}/kata-assets/" < /dev/null diff --git a/hack/release/draft-release.sh b/hack/release/draft-release.sh index d598bc9fea..2bde80d8af 100755 --- a/hack/release/draft-release.sh +++ b/hack/release/draft-release.sh @@ -156,12 +156,13 @@ if [[ "${REPO}" == "${UPSTREAM_REPO}" ]]; then notes+=" [Release images](https://github.com/${REPO}/blob/${tag}/docs/dev/releasing.md#release-images)." notes+=$'\n\n## Install\n\n' notes+="From a checkout of \`${tag}\`, with Go, install the release images into the current kubectl context." - notes+=" Add \`--kind\` for a kind cluster." + notes+=" Add \`--kind\` for a kind cluster, or \`--sandbox-classes=gvisor\` to skip staging microVM guest assets to \`-assets\`." notes+=$'\n\n```sh\n' notes+="git clone --depth 1 --branch ${tag} https://github.com/${REPO}.git"$'\n' notes+=$'cd substrate\n' notes+=$'go run ./cmd/ate-setup deploy ate-system \\\n' notes+=" --image-repo ${RELEASE_IMAGE_REPO} --image-tag ${tag} \\"$'\n' + notes+=$' --storage-bucket-name= \\\n' notes+=$' --credential-provider=\'{"name":"k8s.io"}\' \\\n' notes+=$' --ingress-allowed-spiffe-ids=\n' notes+='```' diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 995cb028f6..373973a43f 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -19,15 +19,13 @@ # and calls this script), mirroring install-ate.sh / install-ate-kind.sh. # # Composes: -# 1. hack/install-ate.sh --deploy-ate-system (control plane) -# 2. hack/install-microvm-deps.sh --install (asset build/stage + cluster-wide -# microvm SandboxConfig) -# 3. Deploy the counter-microvm demo (worker pool manifest, atespace, and -# ActorTemplate through the ate API). +# 1. hack/install-ate.sh deploy ate-system (control plane + microVM assets and SandboxConfig) +# 2. hack/install-ate.sh deploy demo counter-microvm (worker pool manifest, atespace, and +# ActorTemplate through the ate API) # # Like the other hack scripts, this sources .ate-dev-env.sh for the cluster / # registry / bucket settings unless NO_DEV_ENV is set. The control plane deploy -# requires a credential provider selection, which reaches it only through the +# requires a credential provider selection, which reaches it through the # environment here: export ATE_CREDENTIAL_PROVIDER (e.g. '{"name":"k8s.io"}'; # see hack/install-ate.sh --help) before running this, the kind wrapper # included. @@ -35,12 +33,10 @@ # Env (most come from .ate-dev-env.sh): # KO_DOCKER_REPO (required) image registry, e.g. gcr.io/PROJECT/ate-images for # GKE or localhost:5001 for kind. -# BUCKET_NAME object store bucket for snapshots (default: ate-snapshots). On -# GKE the assets go to BUCKET_NAME-assets, see install-microvm-deps.sh. +# BUCKET_NAME object store bucket for snapshots (default: ate-snapshots on kind; +# required on GKE, where ate-setup stages assets to BUCKET_NAME-assets). # KUBECTL_CONTEXT (optional) kube context; threaded into install + ko apply + kubectl. # PROJECT_ID (optional) GCP project for the GCS asset upload (GKE path). -# ARCH target arch (default: from KO_DEFAULTPLATFORMS, else host arch). -# OUT asset dir (default: $PWD/bin/microvm-assets/$ARCH, gitignored). # ATE_INSTALL_KIND "true" for the kind path (stage assets to rustfs + install-ate-kind.sh); # default false uploads assets to GCS + uses install-ate.sh. @@ -58,7 +54,6 @@ fi # --- env / defaults --------------------------------------------------------- KO_DOCKER_REPO="${KO_DOCKER_REPO:-}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" -BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" if [[ $# -gt 0 ]]; then echo "Error: unknown argument $1" >&2 @@ -79,23 +74,17 @@ log() { echo -e "${COLOR_CYAN}[run-microvm-demo]: $*${COLOR_RESET}" } -# --- 1. deploy the control plane ------------------------------------------- +# --- 1. deploy the control plane (includes microVM assets + SandboxConfig) - log "Deploying the ate control plane (deploy ate-system)..." if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. + # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ATE_INSTALL_KIND itself. KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh deploy ate-system else # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh deploy ate-system fi -# --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- -# install-microvm-deps.sh handles the assemble/stage/apply flow. Ordering -# matters: the control plane must be up so the SandboxConfig CRD exists. -log "Installing micro-VM dependencies..." -KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-microvm-deps.sh --install - -# --- 3. apply the demo ------------------------------------------------------ +# --- 2. apply the demo ------------------------------------------------------ KCTX_FLAG="" if [[ -n "${KUBECTL_CONTEXT}" ]]; then KCTX_FLAG=" --context=${KUBECTL_CONTEXT}" @@ -104,7 +93,7 @@ fi # The demo handler applies the worker pool, creates the atespace and the # ActorTemplate through the ate API, and waits for the golden snapshot; # dispatch through install-ate.sh like step 1. -log "Deploying the counter-microvm demo (--deploy-demo-counter-microvm)..." +log "Deploying the counter-microvm demo (deploy demo counter-microvm)..." if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh deploy demo counter-microvm else diff --git a/internal/e2e/suites/egressmitm/egressmitm_test.go b/internal/e2e/suites/egressmitm/egressmitm_test.go index 2d6da343b3..de3e43889c 100644 --- a/internal/e2e/suites/egressmitm/egressmitm_test.go +++ b/internal/e2e/suites/egressmitm/egressmitm_test.go @@ -63,9 +63,6 @@ var probeNamespace string // hack/setup-e2e-egress-tls-kind.sh // hack/run-e2e-kind.sh ./internal/e2e/suites/egressmitm -v -args --no-color // E2E_SANDBOX_CLASS=microvm hack/run-e2e-kind.sh ./internal/e2e/suites/egressmitm -v -args --no-color -// -// The micro-VM variant additionally needs the micro-VM deps installed -// (hack/run-microvm-demo-kind.sh, or hack/install-microvm-deps.sh --install). func TestActorEgressMITMTrust(t *testing.T) { env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") if err != nil { diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl similarity index 53% rename from manifests/microvm/sandboxconfig-microvm.yaml.tmpl rename to manifests/ate-install/sandboxconfig-microvm.yaml.tmpl index 19bd5b6fd3..fdb5d49666 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl @@ -12,28 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Cluster-wide SandboxConfig for the micro-VM (kata + cloud-hypervisor) sandbox -# class. Unlike sandboxconfig-gvisor.yaml (applied unconditionally by -# hack/install-ate.sh --deploy-ate-system), this is opt-in: apply via -# hack/install-microvm-deps.sh --install after staging the asset set -# (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh). A microvm WorkerPool -# resolves it as its class default (or names it via configRef: microvm), so -# a missing/stale config shows on the pool's status. +# Cluster-wide SandboxConfig for the microVM (kata + cloud-hypervisor) sandbox +# class. `go run ./cmd/ate-setup deploy ate-system` (and `deploy sandboxconfig`) +# assembles the asset set, stages it to the cluster's object store, and applies +# this manifest alongside sandboxconfig-gvisor.yaml. A microvm WorkerPool +# resolves it as its class default (or names it via configRef: microvm), so a +# missing/stale config shows on the pool's status. # # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs) # are FETCHED at runtime from the asset bucket ${ASSET_BUCKET_NAME} under -# kata-assets/ (the rustfs bucket on kind, its own GCS bucket on GKE), through -# the node's object-store plugin - NOT baked into the worker image. +# microvm-assets// (the rustfs bucket on kind, its own GCS +# bucket on GKE), through the node's object-store plugin - NOT baked into the +# worker image. # ${BUCKET_SCHEME} is that store's URL scheme (s3 on kind, gs on GKE): the # plugin serves only its own backend's scheme. The assets must stay outside any # snapshot location, which the plugin refuses to serve as assets. ateworker -# boots cloud-hypervisor itself and gives the actor an overlay rootfs (virtio-fs -# RO lower + guest-tmpfs upper), so it fetches virtiofsd too. +# boots cloud-hypervisor itself and gives the actor an overlay rootfs +# (virtio-fs RO lower + guest-tmpfs upper), so it fetches virtiofsd too. # -# The per-arch sha256 values below are the asset sets produced by -# hack/microvm-assets/assemble.sh; atelet selects the block matching the node's -# architecture, and each cluster's bucket holds that arch's binaries at these -# paths (staged by hack/install-microvm-deps.sh --install). +# cmd/ate-setup/internal/steps/microvm.go reads the per-arch sha256 values +# below to verify the assembled binaries and stages them at these digest-keyed +# paths so newer releases do not overwrite kept versions. apiVersion: ate.dev/v1 kind: SandboxConfig @@ -50,29 +49,28 @@ spec: assets: arm64: cloud-hypervisor: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/cloud-hypervisor" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/arm64/f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850" sha256: "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850" # virtiofsd serves the overlay RO lower (virtio-fs). virtiofsd: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/virtiofsd" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/arm64/4b43f4f6de6500203b01214a790314eea4182f5f110594ddb28ae2805622f06f" sha256: "4b43f4f6de6500203b01214a790314eea4182f5f110594ddb28ae2805622f06f" kata-kernel: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/vmlinux" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/arm64/dece388830720fe8bddb41552fff5f486240aeece5acc0aebc1812761e3e3f66" sha256: "dece388830720fe8bddb41552fff5f486240aeece5acc0aebc1812761e3e3f66" kata-image: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/rootfs.img" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/arm64/69f9a76e5a2563b7c74d1f271f3a904240f50ef405465f832021525894519f97" sha256: "69f9a76e5a2563b7c74d1f271f3a904240f50ef405465f832021525894519f97" - # amd64 assets are kata 4.3.1 (assemble.sh ARCH=amd64). amd64: cloud-hypervisor: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/cloud-hypervisor" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/amd64/448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc" sha256: "448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc" virtiofsd: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/virtiofsd" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/amd64/15b2e72a78cc08a9bd8a6943e89fb69c88cb3cbeb63069efceade835342ac7d4" sha256: "15b2e72a78cc08a9bd8a6943e89fb69c88cb3cbeb63069efceade835342ac7d4" kata-kernel: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/vmlinux" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/amd64/f87d8359035cb7557b16a319e6431ea09b183e62268aef436bf8dc6530d786b6" sha256: "f87d8359035cb7557b16a319e6431ea09b183e62268aef436bf8dc6530d786b6" kata-image: - url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/kata-assets/rootfs.img" + url: "${BUCKET_SCHEME}://${ASSET_BUCKET_NAME}/microvm-assets/amd64/6770db2cca5710593f27c9ac5e1d9d7aab836422695a5e5c4807496a92a0f24d" sha256: "6770db2cca5710593f27c9ac5e1d9d7aab836422695a5e5c4807496a92a0f24d" diff --git a/tools/setup-gcp/README.md b/tools/setup-gcp/README.md index f0c685de76..db70000f57 100644 --- a/tools/setup-gcp/README.md +++ b/tools/setup-gcp/README.md @@ -329,7 +329,7 @@ bucket. Upgrade Substrate first, to a release whose node plugin gets snapshot credentials from ate-api-server's credential exchange. On an older release, removing these grants makes every suspend and resume fail with `403`. Remove those grants after running `create bucket` and `create iam`, and after -rerunning `hack/install-microvm-deps.sh --install` if you use microVM +rerunning `go run ./cmd/ate-setup deploy sandboxconfig` if you use microVM workers, so the assets are in the asset bucket. `ATELET` is set as in the manual equivalent above: diff --git a/tools/setup-gcp/cmd/common.go b/tools/setup-gcp/cmd/common.go index 9eb30093f0..4fb7fc45b0 100644 --- a/tools/setup-gcp/cmd/common.go +++ b/tools/setup-gcp/cmd/common.go @@ -123,7 +123,7 @@ func getEnv[T getEnvType](key string, fallback T) T { } // assetBucketName is the bucket for micro-VM sandbox assets. -// hack/install-microvm-deps.sh derives the same name. +// cmd/ate-setup derives the same name. func (c *Config) assetBucketName() string { return c.BucketName + "-assets" }