diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index b320dd3366..26879fe159 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -47,7 +47,7 @@ body: label: Steps to Reproduce description: How can the issue be reproduced? Provide commands, scripts, or manifests (e.g., ActorTemplate). placeholder: | - 1. Deploy Substrate with `hack/install-ate-kind.sh ...` + 1. Deploy Substrate with `go run ./cmd/ate-setup --kind ...` 2. Apply manifest `kubectl apply -f ...` 3. Run command `...` 4. See error diff --git a/.github/SECURITY.md b/.github/SECURITY.md index d5acdcc747..7315142a06 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -37,8 +37,8 @@ include: ## What Counts as a Vulnerability -See the [Security Guide](../docs/security.md) and -[Threat Model](../docs/threat-model.md) for the security architecture, trust +See the [Security Guide](../docs/security/security.md) and +[Threat Model](../docs/security/threat-model.md) for the security architecture, trust boundaries, and assumptions this section is based on. In scope: the Agent Substrate control plane (`ateapi`, `atecontroller`, @@ -81,7 +81,7 @@ from most to least severe: - Sample applications under `demos/` and load-testing harnesses under `benchmarking/`. - Documented scope boundaries and future capabilities listed in - [docs/security.md](../docs/security.md) at the time of your report. If you are + [docs/security/security.md](../docs/security/security.md) at the time of your report. If you are unsure whether a finding is covered by a documented limitation, report it privately anyway and let a maintainer make the call. diff --git a/.github/scripts/release/ghcr-login.sh b/.github/scripts/release/ghcr-login.sh index 9d6dadbbb8..61d9e7ada6 100755 --- a/.github/scripts/release/ghcr-login.sh +++ b/.github/scripts/release/ghcr-login.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Logs docker in to ghcr.io as GITHUB_ACTOR with the token in TOKEN. ko and -# ate-setup's registry checks read the credentials docker login stores. +# Logs docker in to ghcr.io as GITHUB_ACTOR with the token in TOKEN. ko, buildx, +# and ate-setup's registry checks read the credentials docker login stores. # # GITHUB_ACTOR= TOKEN= .github/scripts/release/ghcr-login.sh # diff --git a/.github/workflows/govulncheck.yaml b/.github/workflows/govulncheck.yaml index 35255104c9..0b1bbdd528 100644 --- a/.github/workflows/govulncheck.yaml +++ b/.github/workflows/govulncheck.yaml @@ -32,7 +32,6 @@ jobs: # The root module, then each module kept apart from it. work-dir: - . - - internal/plugins/gcp-secret-manager steps: - id: govulncheck uses: golang/govulncheck-action@032d45514ae346b1db93c04b0c90b841c370344f # v1.1.0 diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 4dbfd3ecb5..62edbf740e 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -74,12 +74,12 @@ jobs: run: helm upgrade --install substrate-crds charts/substrate-crds - name: Bootstrap mTLS authorities run: | - hack/install-ate-kind.sh create podcertificate-controller-cas - hack/install-ate-kind.sh create jwt-authority-pool - hack/install-ate-kind.sh create actor-id-ca-pool - hack/install-ate-kind.sh create actor-id-ca-certs - hack/install-ate-kind.sh create api-authentication-config - hack/install-ate-kind.sh create egress-mitm-ca-pool + go run ./cmd/ate-setup --kind create podcertificate-controller-cas + go run ./cmd/ate-setup --kind create jwt-authority-pool + go run ./cmd/ate-setup --kind create actor-id-ca-pool + go run ./cmd/ate-setup --kind create actor-id-ca-certs + go run ./cmd/ate-setup --kind create api-authentication-config + go run ./cmd/ate-setup --kind create egress-mitm-ca-pool - name: Install certificate controller run: | helm upgrade --install substrate-podcert charts/substrate-podcert \ @@ -100,7 +100,9 @@ jobs: --set image.registry=localhost:5001 \ --set image.tag=helm-e2e \ --set 'preview[0]=ALL' \ + --set 'atenetRouter.ingressAllowedSPIFFEIDs[0]=spiffe://cluster.local/ns/ate-e2e-router-client/sa/router-client' \ --set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \ + --set 'atelet.extraArgs[1]=--snapshot-upload-delay=2s' \ --set otel.endpoint=http://opentelemetry-collector.otel-system.svc:4317 \ --wait --timeout=10m - name: Enable NFS @@ -108,16 +110,16 @@ jobs: sudo modprobe nfs || true sudo modprobe nfsd || true - name: Install CSI NFS driver - run: hack/install-ate-kind.sh setup csi nfs + run: go run ./cmd/ate-setup --kind setup csi nfs - name: Deploy micro-VM counter demo # The deploy creates the substrate ActorTemplate and waits for its golden # snapshot internally; the ActorTemplate CRD (and its Ready condition) # no longer exists to wait on. run: hack/run-microvm-demo-kind.sh --skip-control-plane - name: Deploy gVisor counter demo - run: hack/install-ate-kind.sh deploy demo counter + run: go run ./cmd/ate-setup --kind deploy demo counter - name: Deploy egress demo - run: hack/install-ate-kind.sh deploy demo egress + run: go run ./cmd/ate-setup --kind deploy demo egress - name: Trust local E2E TLS origins run: hack/setup-e2e-egress-tls-kind.sh - name: Run E2E tests (gVisor) diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 3cff0baf96..97d2912653 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -85,22 +85,6 @@ jobs: --jsonfile "${ARTIFACTS}/junittool.json" \ --format pkgname-and-test-fails --format-hide-empty-pkg \ -- -race ./... - # Kept apart from the root module so it can move to its own repository. - - name: gcp-secret-manager credential provider tests - working-directory: internal/plugins/gcp-secret-manager - run: | - go -C ../../../tools/junittool run . register "${ARTIFACTS}/gcp-secret-manager.xml" - ../../../hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/gcp-secret-manager.xml" \ - --jsonfile "${ARTIFACTS}/gcp-secret-manager.json" \ - --format pkgname-and-test-fails --format-hide-empty-pkg \ - -- -race ./... - # The root verifiers stop at the module's go.mod, so it gets its own run: - # gofmt, go vet and the tidy check, then golangci-lint with the root config. - - name: gcp-secret-manager credential provider verify - working-directory: internal/plugins/gcp-secret-manager - run: | - make verify - ../../../hack/run-tool.sh golangci-lint run --config ../../../.golangci.yaml ./... - name: Cache cargo uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: @@ -173,6 +157,9 @@ jobs: preview: ['off', 'on'] env: E2E_ATENET_DATAPLANE: ${{ matrix.dataplane }} + # Apply the e2e test component (short atelet snapshot delays) across the + # initial install and the control plane redeploy in run-microvm-demo-kind.sh. + ATE_INSTALL_E2E_TUNING: "true" # The installer requires a credential provider selection, and deploys the # bundled Kubernetes Secrets provider with the control plane. Job-wide # because every ate-setup invocation reads it, including the control @@ -262,55 +249,28 @@ jobs: # gateway. ATE_PREVIEW above selects the preview features. env: ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} - NO_DEV_ENV: "true" - KO_DOCKER_REPO: "localhost:5001" - ATE_INSTALL_KIND: "true" - BUCKET_NAME: "ate-snapshot" - KIND_CLUSTER_NAME: "kind" - KUBECTL_CONTEXT: "kind-kind" - run: go run ./cmd/ate-setup deploy ate-system --rollout-timeout=300s + run: go run ./cmd/ate-setup --kind deploy ate-system --rollout-timeout=300s - name: Enable NFS # Load NFS kernel modules so in-cluster NFS server and CSI driver can run. run: | sudo modprobe nfs || true sudo modprobe nfsd || true - name: Install CSI NFS driver - env: - NO_DEV_ENV: "true" - KO_DOCKER_REPO: "localhost:5001" - ATE_INSTALL_KIND: "true" - BUCKET_NAME: "ate-snapshot" - KIND_CLUSTER_NAME: "kind" - KUBECTL_CONTEXT: "kind-kind" - run: go run ./cmd/ate-setup setup csi nfs + run: go run ./cmd/ate-setup --kind setup csi nfs - name: Deploy micro-VM counter demo # Stages the (cached) assets into the cluster's rustfs and deploys the # counter-microvm demo onto the control plane installed above. run: hack/run-microvm-demo-kind.sh --skip-control-plane - name: Deploy gVisor counter demo - env: - NO_DEV_ENV: "true" - KO_DOCKER_REPO: "localhost:5001" - ATE_INSTALL_KIND: "true" - BUCKET_NAME: "ate-snapshot" - KIND_CLUSTER_NAME: "kind" - KUBECTL_CONTEXT: "kind-kind" - run: go run ./cmd/ate-setup deploy demo counter + run: go run ./cmd/ate-setup --kind deploy demo counter - name: Deploy egress demos # TestActorEgress in the networking suite builds its Actor from the egress # ActorTemplate for the class under test, so both fixtures have to exist # before their respective lane runs. The demo's actors trust the CA the # gateway mints its leaves from. - env: - NO_DEV_ENV: "true" - KO_DOCKER_REPO: "localhost:5001" - ATE_INSTALL_KIND: "true" - BUCKET_NAME: "ate-snapshot" - KIND_CLUSTER_NAME: "kind" - KUBECTL_CONTEXT: "kind-kind" run: | - go run ./cmd/ate-setup deploy demo egress - go run ./cmd/ate-setup deploy demo egress-microvm + go run ./cmd/ate-setup --kind deploy demo egress + go run ./cmd/ate-setup --kind deploy demo egress-microvm - name: Trust local TLS origins in egress gateway run: hack/setup-e2e-egress-tls-kind.sh # scale the idle sandbox class's demo pools to zero so each lane has the node diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b1b622c544..c14d6448e0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -80,9 +80,15 @@ jobs: # These jobs can write packages or releases, so they restore no cache # that another workflow could have written. cache: false - # Every release image is built with ko, which cross-compiles Go for arm64 - # without emulation, so the job needs neither QEMU nor buildx. ko and - # ate-setup's registry checks read the credentials docker login stores. + # ko cross-compiles Go for arm64, and the envoy-dataplane Dockerfile + # cross-compiles its Rust module, so the job needs no QEMU. buildx builds + # envoy-dataplane for both platforms. ko, buildx, and ate-setup's registry + # checks read the credentials docker login stores. BuildKit is pinned by + # digest because it builds a published image with the ghcr.io login. + - name: Set up buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + with: + driver-opts: image=moby/buildkit:v0.33.1@sha256:cec9f139f45e93c5c69c60f8b07cfad9f43f4ef6b6a6cd917527fea5ff2e3dea - name: Log in to ghcr.io env: TOKEN: ${{ github.token }} @@ -129,6 +135,17 @@ jobs: # These jobs can write packages or releases, so they restore no cache # that another workflow could have written. cache: false + # The same BuildKit as the images job. Host networking and plain HTTP let + # buildx push to the registry service. + - name: Set up buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + with: + driver-opts: | + image=moby/buildkit:v0.33.1@sha256:cec9f139f45e93c5c69c60f8b07cfad9f43f4ef6b6a6cd917527fea5ff2e3dea + network=host + buildkitd-config-inline: | + [registry."localhost:5000"] + http = true - name: Publish images run: go run ./cmd/ate-setup publish release-images draft: diff --git a/.github/workflows/site.yaml b/.github/workflows/site.yaml new file mode 100644 index 0000000000..efca938169 --- /dev/null +++ b/.github/workflows/site.yaml @@ -0,0 +1,120 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds the website in site/ on every pull request that could change it, and +# publishes it to GitHub Pages from main. +name: site +on: + pull_request: + paths: + - 'site/**' + - 'docs/**' + - 'demos/*/README.md' + - 'tools/site-content/**' + - 'tools/site-release-data/**' + - 'hack/site.sh' + - 'hack/test-site-templates.sh' + - '.github/workflows/site.yaml' + push: + branches: [main] + paths: + - 'site/**' + - 'docs/**' + - 'demos/*/README.md' + - 'tools/site-content/**' + - 'tools/site-release-data/**' + - 'hack/site.sh' + - 'hack/test-site-templates.sh' + - '.github/workflows/site.yaml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +env: + # Keep in sync with module.hugoVersion.min in site/hugo.yaml. + HUGO_VERSION: 0.163.3 +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + # site-release-data needs the latest release tag and its history. + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + # The root module, for tools/site-release-data. Hugo fetches Docsy + # with the same Go. + go-version-file: 'go.mod' + cache: false + - name: Setup Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + cache: npm + cache-dependency-path: site/package-lock.json + - name: Install Hugo + run: | + base="https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}" + tarball="hugo_extended_${HUGO_VERSION}_linux-amd64.tar.gz" + cd "${RUNNER_TEMP}" + curl -sSfLO "${base}/${tarball}" + curl -sSfLO "${base}/hugo_${HUGO_VERSION}_checksums.txt" + grep " ${tarball}\$" "hugo_${HUGO_VERSION}_checksums.txt" | sha256sum -c - + mkdir -p hugo && tar -xzf "${tarball}" -C hugo hugo + echo "${RUNNER_TEMP}/hugo" >> "${GITHUB_PATH}" + - name: Install PostCSS + working-directory: site + run: npm ci + - name: Generate site content + run: hack/site.sh content + - name: Generate release data + run: hack/site.sh release-data + - name: Test templates + if: github.event_name == 'pull_request' + run: hack/test-site-templates.sh + - name: Configure Pages + id: pages + if: github.event_name != 'pull_request' + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + - name: Build + working-directory: site + env: + BASE_URL: ${{ steps.pages.outputs.base_url }} + run: hugo build --environment production --minify ${BASE_URL:+--baseURL "${BASE_URL}/"} + - name: Upload artifact + if: github.event_name != 'pull_request' + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + path: site/public + deploy: + if: github.event_name != 'pull_request' + needs: build + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/AGENTS.md b/AGENTS.md index 75833863d9..3919a29188 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,8 +2,9 @@ ## Rules for contributors -- Read the "Using AI" section of `COLLABORATING.md`. You are responsible for everything your agent submits. -- Say in the PR description that you used AI, and how. Don't add AI co-author or assisted-by trailers. Submit commits and PRs from your own account. We can't accept contributions from agent or bot accounts (a human must sign the contribution agreement). +- Read `CONTRIBUTING.md` and the "Using AI" section of `COLLABORATING.md`. You are responsible for everything your agent submits. +- Say in the PR description that you used AI, and how. Don't add AI co-author or assisted-by trailers. Submit commits and PRs from your own account. We can't accept contributions from agent or bot accounts. +- Agents must never sign off a commit as themselves. Only a human can certify the [DCO](https://developercertificate.org/). - Before asking for review, review your own diff. For substantial changes, also have a fresh agent session review it adversarially, then validate its findings and fix the confirmed issues. - **Be brief.** Code comments, commit messages, PR descriptions and issues should be short, plain English, and read the way an experienced maintainer writes by hand. Reviewer time and agent context windows are limited. - `COLLABORATING.md` doesn't allow AI-generated commit messages, because they have usually been low quality and unreviewed. If you are asked to write one anyway, hold it to the same standard, and the person submitting it must review it. @@ -65,7 +66,7 @@ It takes the Kubernetes control-plane out of the critical path to achieve lower Agent Substrate relies on the fact that agent-like applications tend to be idle most of the time to achieve heavy multiplexing. For development, it's recommended to read the `README.md` and `CONTRIBUTING.md` in the root folder. -See `hack/install-ate.sh` and `tools/setup-gcp` for provisioning and deploying clusters and GCP resources. +See `cmd/ate-setup` and `tools/setup-gcp` for provisioning and deploying clusters and GCP resources. ## Repository Layout @@ -94,7 +95,7 @@ tools/ # Standalone Go tools (go run ./tools/) for Dev/CI | Internal proto (atelet / ateworker) | `internal/proto/` | | Dev/CI scripts | `hack/` | | Standalone Go dev/CI tools | `tools/` with its own `go.mod` | -| Self-contained plugin that nothing in the repo depends on | `internal/plugins/` with its own `go.mod`, importing only `api/` and `pkg/`; see `internal/plugins/README.md` | +| Plugin for a specific backend, implementing a plugin API like `api/credprovider` | Its own repository in the `agent-substrate-plugins` org; see "Plugins" in `docs/dev/code-layout.md` | See `docs/dev/code-layout.md` for the full rationale and per-directory details. @@ -103,7 +104,7 @@ See `docs/dev/code-layout.md` for the full rationale and per-directory details. The dev entry points are the scripts under `hack/` and the `cmd/ate-setup` installer. ### Building and Installing -- **Install from source**: `go run ./cmd/ate-setup deploy ate-system` (or the `hack/install-ate.sh` shim) builds every image with ko as part of the install; see `cmd/ate-setup/README.md` +- **Install from source**: `go run ./cmd/ate-setup deploy ate-system` builds every image with ko as part of the install; see `cmd/ate-setup/README.md` - **Binaries**: `go build -o bin/kubectl-ate ./cmd/kubectl-ate`, `go build -o bin/ate-setup ./cmd/ate-setup` - **Release images**: `VERSION=TAG go run ./cmd/ate-setup publish release-images --ko-docker-repo REPO` (the release set, `images.ReleaseComponents`, all tagged `TAG`; see `docs/dev/releasing.md`) - **Release binaries**: `VERSION=TAG go run ./cmd/ate-setup publish release-binaries --out DIR` (`kubectl-ate` for linux and darwin, amd64 and arm64, with `.sha256` files) @@ -129,6 +130,10 @@ Agent Substrate makes no compatibility guarantee before v1.0.0. Do not add code Remove or rename fields, RPCs, flags, and types outright and update every caller in the same change. Do not add `reserved` statements to protos, deprecated aliases, fallbacks for old formats, or version negotiation. +## Version Skew + +From v1.0.0, changes must keep upgrades and rollbacks working under [docs/reference/version-skew-policy.md](docs/reference/version-skew-policy.md). Read it before you change an RPC, a CRD, the worker pod template, a database migration, or file(s) that `atelet` and `ateworker` share on a node. + ## Commit Messages - **Describe the change and why it was needed**: The message is read on `main` long after the PR branch is gone, so it should stand on its own. @@ -144,7 +149,7 @@ Some attributes are defined in the registry but barred from metric labels: actor `docs/metrics/substrate.yaml` holds the rules Weaver cannot express: the cardinality rules, the known exceptions, and the subsystems that emit no metrics. Read `blind_spots` before you attribute a fault to a component. -See the [metric registry](docs/observability.md#the-metric-registry) section of the observability guide. +See the [metric registry](docs/guides/observability.md#the-metric-registry) section of the observability guide. ## Testing Instructions @@ -156,9 +161,9 @@ See the [metric registry](docs/observability.md#the-metric-registry) section of ## Security Considerations -Agent Substrate is built for zero-trust, multi-tenant execution of untrusted actors. Preserve and extend these invariants when modifying the codebase (see `docs/security.md` and `docs/threat-model.md` for the full architecture): +Agent Substrate is built for zero-trust, multi-tenant execution of untrusted actors. Preserve and extend these invariants when modifying the codebase (see `docs/security/security.md` and `docs/security/threat-model.md` for the full architecture): - **Workload & Storage Isolation**: Actors run strictly inside `gVisor` (`cmd/ateworker-gvisor`) or `Cloud Hypervisor` microVM (`cmd/ateworker-microvm`) sandboxes; plain Linux containers are not supported. `atelet` scrubs actor directories across lifecycle transitions (`resetActorDirs`), and snapshots are stamped with `ActorSandboxRuntime` metadata and transferred via `cmd/objectstore-plugin`. - **Component & Actor Identity (mTLS, JWTs, and JWKs)**: Inter-component communication uses mTLS backed by `cmd/podcertcontroller` (`podid`, `servicedns`, and `postgres` signers). Actor JWT signing (`internal/localjwtauthority`), verification (`internal/oidcjwt`), and key discovery (served via `cmd/ate-idp-server`) are built on the standard library's crypto packages. Extend those packages rather than adding `go-jose` or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses. -- **Control-Plane Authentication & Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), always enforced. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/authorization.md` for roles and known gaps. Callers are authenticated via `SPIFFE` mTLS or configured OIDC `JWTProvider`s (`cmd/ateapi/internal/apiauthn`). +- **Control-Plane Authentication & Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), always enforced. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/guides/authorization.md` for roles and known gaps. Callers are authenticated via `SPIFFE` mTLS or configured OIDC `JWTProvider`s (`cmd/ateapi/internal/apiauthn`). - **Network Isolation, Ingress, & Egress**: `manifests/ate-install/ate-system-networkpolicies.yaml` enforces default-deny ingress across `ate-system`, and `cmd/atecontroller` reconciles per-`WorkerPool` `NetworkPolicy` objects. Actors run in private `/30` point-to-point veth network namespaces (`internal/ateworker/actornet`) with link-local blocking, policy-enforced DNS filtering (`internal/atunnel/dns`), and zero ambient credentials. Outbound traffic traverses `atunnel` to `atenet-egress`, which enforces per-actor `EgressPolicy` and injects credentials out-of-band via `cmd/credential-provider/kubernetes-secrets` (`k8s-credential-provider`) or actor JWT minting. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6b716b883f..996d14f619 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,121 +1,3 @@ # How to Contribute -We would love to accept your patches and contributions to this project. Before -you spend a lot of time on a contribution, please review the following -guidelines. - -## Before you begin - -### Sign our Contributor License Agreement - -Contributions to this project must be accompanied by a -[Contributor License Agreement](https://cla.developers.google.com/about) (CLA). -You (or your employer) retain the copyright to your contribution; this simply -gives us permission to use and redistribute your contributions as part of the -project. - -If you or your current employer have already signed the Google CLA (even if it -was for a different project), you probably don't need to do it again. - -Visit to see your current agreements or to -sign a new one. - -### Review our Community Guidelines - -This project follows [Google's Open Source Community -Guidelines](https://opensource.google/conduct/). - -### Set up a local development environment - -The [Quickstart (Development)](README.md#quickstart-development) in the README -covers bringing up a local cluster with the default (gVisor) runtime. To run -the microVM runtime locally — which needs `/dev/kvm`, or Lima nested -virtualization on Apple Silicon — see -[docs/dev/microvm-local.md](docs/dev/microvm-local.md). To measure where a -suspend or resume spends its time on such a cluster, follow -[docs/dev/suspend-resume-phase-breakdown.md](docs/dev/suspend-resume-phase-breakdown.md). - -## Contribution process - -This is a very new project, so we are still working out exactly how it is going -to be developed. For now, we are focused on iterating quickly to find the right -design and architecture. This has implications for contributors: - -1) Things are moving quickly, so PRs may need to be rebased or updated -frequently. Small PRs that are focused on a single issue or feature are easier -to review and update than large PRs that touch many different parts of the -codebase. - -2) While we welcome new contributors, we are really focused on the minimal -capabilities needed to make this project useful. Small fixes and cleanups can -go straight to a PR. Before you start anything larger, please discuss it with -us first (if there is an issue open, comment there and if not, open one), and -search open PRs so we are not duplicating work that is already in flight. We -want to make sure that your work is aligned with our near-term goals for the -project. - -3) PRs which are not aligned with our near-term goals may be closed without -extensive review. We are not trying to be discouraging, but we need to make -sure that we are focused on the most important work. - -If you are building something that runs *on* Substrate rather than changing -Substrate itself, see [Integration -Repositories](docs/integration-repos.md) for where that code should live. - -### Sizing PRs for review - -We optimize PRs for easy review — large PRs get broken -down, small PRs get merged. - -* **Large PRs**: split huge changes into a series of smaller PRs, each a - logically distinct feature. When the intermediate steps are not useful - on their own, keep the change as one PR split into commits at logical - break points for review. PRs are squash-merged, so only the branch - `HEAD` needs to build and pass tests. -* **Small and bulk PRs**: if you find a typo, review the whole file and - fix everything in one pass rather than sending the single edit. Group - related typo, doc, and single-line cleanup fixes into one PR rather - than opening several small ones for the same area. Maintainers may ask - you to consolidate fragmented PRs into one, or close them in favor of a - combined submission. - -As a rough scale: S is under 30 changed lines, M under 100, L under 500, XL under 1000. Most PRs should be L or smaller; XL and above are candidates for breaking down. - -### Code Reviews - -All submissions, including submissions by project members, require review. We -use [GitHub pull requests](https://docs.github.com/articles/about-pull-requests) -for this purpose. - -Changes that add or alter behavior should come with tests, and existing tests -should be updated when behavior they cover changes. We will not merge code that -causes tests to fail. Every test runs in CI on every PR, so please don't add -tests whose only purpose is to raise coverage numbers. Tests should be -inherently valuable for future changes. - -Follow the [PostgreSQL schema evolution rules](docs/dev/postgresql-schema-evolution.md) for each application schema change. - -Each pull request title becomes its line in the [release notes](docs/dev/release-notes.md). - -### Root-gated tests - -Tests that need root (overlay mounts, mknod, `trusted.*` xattrs, ...) call -`roottest.Require(t, ...)` from [internal/roottest](internal/roottest) as -their first statement. They skip in a plain `go test ./...`; CI reruns every -package whose tests import that package under `sudo`. To run them locally: - -```sh -hack/run-root-tests.sh -``` - -New privileged tests only need the `roottest.Require` call — no CI changes. - -### Copyright Headers - -Every file containing source code must include copyright and license -information. This includes any JS/CSS files that you might be serving out to -browsers. (This is to help well-intentioned people avoid accidental copying -that doesn't comply with the license.) - -Our standard headers for various filetypes can be found in -[./hack/boilerplate](./hack/boilerplate/). +Moved to [docs/project/contributing.md](docs/project/contributing.md). diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 6efe908911..d3a3dbdff3 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -20,16 +20,16 @@ decided among current Maintainers. **Default** — No formal designation. Anyone may open issues and PRs on the project, or review other people's PRs and issues. Must follow the Code of -Conduct. Opening a PR requires that the author sign the [Google Contributor -License Agreement](https://cla.developers.google.com/about) (see +Conduct. Every commit in a PR must be signed off under the [Developer +Certificate of Origin](https://developercertificate.org/) (see [CONTRIBUTING.md](CONTRIBUTING.md)). **Contributor** — Someone who has made at least one non-trivial contribution and is known to or vouched for by a Maintainer, or has accumulated enough contributions over time. Granted `Read` access on the repository. Must follow -the Code of Conduct and sign the [Google Contributor License -Agreement](https://cla.developers.google.com/about) before their first -contribution is merged (see [CONTRIBUTING.md](CONTRIBUTING.md)). +the Code of Conduct and sign off every commit under the [Developer Certificate +of Origin](https://developercertificate.org/) (see +[CONTRIBUTING.md](CONTRIBUTING.md)). **Reviewer** — A contributor who has made enough contributions and is vouched for by at least 2 Maintainers. Granted `Triage` access: can review, label, and diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 93767601fe..667cfa2e22 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -20,7 +20,7 @@ The current maintainers of the Agent Substrate project are listed below. | Maya Wang | mayawang | Google | | Michelle Au | msau42 | Google | | Jaana Dogan | rakyll | Google | -| | sfunkenhauser | Google | +| Scott Funkenhauser | sfunkenhauser | Google | | Tim Hockin | thockin | Google | | Zoe Zhao | zoez7 | Google | diff --git a/README.md b/README.md index 22ec50d785..d6dd7a40ea 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[Quickstart](#quickstart-development) · [Security](docs/security.md) · [Architecture](docs/architecture.md) · [Contributing](CONTRIBUTING.md) +[Quickstart](#quickstart-development) · [Security](docs/security/security.md) · [Architecture](docs/concepts/architecture.md) · [Contributing](CONTRIBUTING.md) @@ -25,12 +25,12 @@ Substrate is a low-opinion runtime, not an agent SDK. The workloads it runs do n ## Security Model -Substrate treats every actor as untrusted and mutually hostile, so tenants can share physical nodes without sharing a security boundary. Every control operates within the cold-start and resume budget. See the [Security Guide](docs/security.md) for the full list of security principles and threat model. +Substrate treats every actor as untrusted and mutually hostile, so tenants can share physical nodes without sharing a security boundary. Every control operates within the cold-start and resume budget. See the [Security Guide](docs/security/security.md) for the full list of security principles and threat model. * **Always sandboxed, VM-grade boundaries:** Every actor runs inside a dedicated [gVisor](https://gvisor.dev/) (`runsc`) or [Cloud Hypervisor](https://www.cloudhypervisor.org/) microVM sandbox with segregated compute, network, and storage. Plain Linux containers are not supported. * **Per-agent cryptographic identity:** Every active actor receives a short-lived [SPIFFE](https://spiffe.io/) X.509 certificate by default, and the egress gateway can mint per-actor JWTs on demand when an egress policy calls for one. Both carry the actor identity and atespace (the Substrate namespace grouping), and are designed to federate with external identity systems. -* **Zero credentials in the sandbox:** Actors never hold actor identity JWTs/certs, API keys, or cloud tokens. The built-in egress gateway ([`atenet-egress`](docs/egress-credential-injection.md)) injects credentials into approved outbound requests outside the sandbox, so a prompt-injected or compromised agent has nothing to exfiltrate. -* **Default-deny networking:** Actors start with a private network stack, no inbound path except through the ingress router (`atenet-router`), no lateral actor-to-actor or control-plane access, and outbound traffic [restricted by the egress gateway](docs/egress-traffic.md). +* **Zero credentials in the sandbox:** Actors never hold actor identity JWTs/certs, API keys, or cloud tokens. The built-in egress gateway ([`atenet-egress`](docs/guides/egress-credential-injection.md)) injects credentials into approved outbound requests outside the sandbox, so a prompt-injected or compromised agent has nothing to exfiltrate. +* **Default-deny networking:** Actors start with a private network stack, no inbound path except through the ingress router (`atenet-router`), no lateral actor-to-actor or control-plane access, and outbound traffic [restricted by the egress gateway](docs/guides/egress-traffic.md). * **Pluggable policy at the edge:** The egress gateway acts as an external Policy Enforcement Point (PEP) that is fully replaceable, allowing you to attach your own authorization, DLP, prompt-injection scanning, or cloud IAM policy engine without placing policy code inside the untrusted sandbox. ## Performance & Multiplexing @@ -115,14 +115,19 @@ To quickly set up the complete environment: 2. Run the following steps: ```shell -# create cluster and local registry (IPv4; IP_FAMILY=dual|ipv6 overrides) +# create cluster and local registry hack/create-kind-cluster.sh -# install ate, PostgreSQL, rustfs -hack/install-ate-kind.sh deploy ate-system --credential-provider='{"name":"k8s.io"}' +# make ate-setup target the kind cluster; set this in every shell that runs +# ate-setup, including for the demo walkthroughs +export ATE_INSTALL_KIND=true + +# install Substrate +go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-system/sa/atenet-client # install counter demo -hack/install-ate-kind.sh deploy demo-counter +go run ./cmd/ate-setup deploy demo counter # install kubectl-ate go install ./cmd/kubectl-ate @@ -131,21 +136,20 @@ go install ./cmd/kubectl-ate # actor template, resolved in the actor's atespace) kubectl ate create actor my-counter-1 -a ate-demo-counter --template counter -# port-forward the network router to bind to local port `8000` -kubectl port-forward -n ate-system svc/atenet-router 8000:80 +# serve a local proxy on port 8000 that sends requests to the actor through the +# network router +kubectl ate proxy actor my-counter-1 -a ate-demo-counter ``` > [!IMPORTANT] > `{"name":"k8s.io"}` deploys the bundled credential provider with an empty, > default-deny namespace policy. Before an actor can have a credential > injected, grant its atespace the namespaces whose Secrets it may read; see -> [Enable it](docs/egress-credential-injection.md#enable-it). +> [Enable it](docs/guides/egress-credential-injection.md#enable-it). 3. In a **separate terminal**, send an HTTP request to increment the counter: ```shell -curl -X POST \ - -H "ate-target-actor: ate-demo-counter/my-counter-1" \ - -i http://localhost:8000/ +curl -X POST -i http://localhost:8000/ ``` ### GKE Quickstart (Development) @@ -172,28 +176,36 @@ curl -X POST \ that ate-api-server and atelet depend on. See [what `create iam` actually grants](tools/setup-gcp/README.md#what-create-iam-actually-grants) to audit them or apply them manually. If you bring your own cluster instead, - note the required Kubernetes beta APIs can only be enabled **at cluster - creation** — see the [Create Cluster warning](tools/setup-gcp/README.md#2-create-cluster). + it needs the PodCertificateRequest and ClusterTrustBundle APIs, which GKE + 1.37 and higher serve by default. On 1.36, enable them as beta APIs, then + replace any 1.36 node pools created before that. See the + [Create Cluster warning](tools/setup-gcp/README.md#2-create-cluster). 4. Deploy the Agent Substrate system to your cluster: ```bash - go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}' + go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-system/sa/atenet-client ``` + `--ingress-allowed-spiffe-ids` lists the clients the router lets reach + actors. The ID above is the one `kubectl ate proxy actor` presents. Without + the flag no client reaches actors. See + [Ingress to actors](docs/guides/operator-install.md#ingress-to-actors). + `--credential-provider` is required. `{"name":"k8s.io"}` also deploys the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off - (see [docs/egress-credential-injection.md](docs/egress-credential-injection.md)). + (see [docs/guides/egress-credential-injection.md](docs/guides/egress-credential-injection.md)). `ATE_CREDENTIAL_PROVIDER` in `.ate-dev-env.sh` is the same choice. **Credential injection needs a namespace policy.** The bundled provider starts with an empty, default-deny one, so it resolves no Secrets until you grant each atespace the namespaces it may read; see - [Enable it](docs/egress-credential-injection.md#enable-it). + [Enable it](docs/guides/egress-credential-injection.md#enable-it). 5. You can then deploy the sample applications. See [demos/counter/README.md](demos/counter/README.md) or [demos/sandbox/README.md](demos/sandbox/README.md) for detailed walkthroughs. ```bash - go run ./cmd/ate-setup deploy demo demo-counter + go run ./cmd/ate-setup deploy demo counter ``` #### Custom Setup and Deployment @@ -207,8 +219,8 @@ go run ./tools/setup-gcp create bucket To run the PostgreSQL store backend on Cloud SQL — with IAM database authentication and no passwords — see [tools/setup-gcp/cloud-sql.md](tools/setup-gcp/cloud-sql.md). -Similarly, you can deploy or cleanup specific Agent Substrate components using -the installation script. See `go run ./cmd/ate-setup` for all options. +Similarly, you can deploy or clean up specific Agent Substrate components with +`ate-setup`. See `go run ./cmd/ate-setup --help` for all options. ```bash # Re-deploy only ate-apiserver of the ATE system go run ./cmd/ate-setup deploy apiserver @@ -222,9 +234,12 @@ go run ./cmd/ate-setup delete all If you need to delete the resources created by the setup script, you can use the provided script `hack/teardown.sh`. This script will delete resources in the reverse order of creation and handles partial failures gracefully. ```bash -./hack/teardown.sh --all +./hack/teardown.sh [--all|--cluster-scoped] ``` +* `--all` deletes everything, including the dashboards, the snapshot transfer service account and the project-level IAM bindings, which every Substrate cluster in the project shares. +* `--cluster-scoped` deletes only this cluster, its buckets and the bindings on them. Use it while other Substrate clusters still use the project. + Or run individual teardown steps as needed (see `./hack/teardown.sh` for available options). #### Tearing down local `kind` resources @@ -246,21 +261,22 @@ We provide several sample applications demonstrating Agent Substrate's capabilit 5. **[Request Parking](demos/parking/README.md)**: An oversubscribed pool where the router holds inbound requests until a worker frees up, instead of returning `503`. ### Documentation & Guides -* [Architecture](docs/architecture.md): How the control plane, node supervisor, and networking stack fit together. -* [API Configuration Guide](docs/api-guide.md): Detailed reference for configuring WorkerPools, ActorTemplates, Secrets, and Volumes. -* [Full CLI Documentation](cmd/kubectl-ate/README.md): Installation and usage for `kubectl-ate`. -* [Glossary](docs/glossary.md): Core terms (Actor, Atespace, ActorTemplate, WorkerPool, Worker, ate-api-server, atenet, atelet, ateworker) and how they relate. -* [Integration Repositories](docs/integration-repos.md): Where integrations live, how their repositories are named, and how fixes flow back to core. -* [Observability Guide](docs/observability.md): Guide to actor logging, metrics, and distributed tracing. -* [Authentication Guide](docs/authentication.md): Configure trusted JWT providers and human credentials. -* [Egress Traffic](docs/egress-traffic.md): Which protocols an Actor may reach the outside world with and which are blocked. -* [Enabling man-in-the-middle (MITM) interception for Actor Egress policy](docs/egress-trust-bundle.md): Egress policies such as header injection depend on MITM interception of Actor traffic. This guide explains how an Actor should be configured to enable interception. -* [Request Parking](docs/request-parking.md): How the router parks requests through transient worker-pool saturation. -* [Sandbox Asset Cache](docs/sandbox-asset-cache.md): How a node caches the sandbox binaries actors run on, decides which runtimes it needs, and evicts to a budget. -* [Upgrade Runbook](docs/upgrade.md): Upgrade a running install to a newer supported release, and roll back if needed. -* [Node Storage](docs/node-storage.md): How to give snapshot I/O its own disk. -* [Security Guide](docs/security.md): Security overview, security principles, and threat modeling. -* [Benchmarking Guide](benchmarking/README.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness. +* [Architecture](docs/concepts/architecture.md): How the control plane, node supervisor, and networking stack fit together. +* [API Configuration Guide](docs/reference/api-guide.md): Detailed reference for configuring WorkerPools, ActorTemplates, Secrets, and Volumes. +* [Full CLI Documentation](docs/reference/kubectl-ate.md): Installation and usage for `kubectl-ate`. +* [Glossary](docs/reference/glossary.md): Core terms (Actor, Atespace, ActorTemplate, WorkerPool, Worker, ate-api-server, atenet, atelet, ateworker) and how they relate. +* [Integration Repositories](docs/project/integration-repos.md): Where integrations live, how their repositories are named, and how fixes flow back to core. +* [Observability Guide](docs/guides/observability.md): Guide to actor logging, metrics, and distributed tracing. +* [Extending Substrate](docs/security/extensibility.md): How to augment or replace the built-in ingress and egress gateways. +* [Authentication Guide](docs/guides/authentication.md): Configure trusted JWT providers and human credentials. +* [Egress Traffic](docs/guides/egress-traffic.md): Which protocols an Actor may reach the outside world with and which are blocked. +* [Enabling man-in-the-middle (MITM) interception for Actor Egress policy](docs/guides/egress-trust-bundle.md): Egress policies such as header injection depend on MITM interception of Actor traffic. This guide explains how an Actor should be configured to enable interception. +* [Request Parking](docs/concepts/request-parking.md): How the router parks requests through transient worker-pool saturation. +* [Sandbox Asset Cache](docs/concepts/sandbox-asset-cache.md): How a node caches the sandbox binaries actors run on, decides which runtimes it needs, and evicts to a budget. +* [Upgrade Runbook](docs/guides/upgrade.md): Upgrade a running install to a newer supported release, and roll back if needed. +* [Node Storage](docs/guides/node-storage.md): How to give snapshot I/O its own disk. +* [Security Guide](docs/security/security.md): Security overview, security principles, and threat modeling. +* [Benchmarking Guide](docs/guides/benchmarking.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness. ## Tour @@ -275,7 +291,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * `cmd/ateworker-microvm`: The microVM peer of `ateworker-gvisor`, running actors as cloud-hypervisor VMs. * `cmd/podcertcontroller`: A "polyfill" that provides Pod Certificate signers that will eventually ship in upstream Kubernetes (with different names). -* `cmd/kubectl-ate`: A CLI tool for managing Agent Substrate resources. See its [README](cmd/kubectl-ate/README.md). +* `cmd/kubectl-ate`: A CLI tool for managing Agent Substrate resources. See its [README](docs/reference/kubectl-ate.md). * `cmd/benchmarking`: Synthetic workloads used by the load tests, including `glutton`, which consumes RAM, disk, and file descriptors on demand. * `tools/setup-gcp`: A provisioning utility to set up the necessary GCP infrastructure resources (GKE, GCS, IAM). * `demos/`: Sample applications demonstrating Agent Substrate capabilities. diff --git a/api/ateapi/v1/AGENTS.md b/api/ateapi/v1/AGENTS.md index 511e64317d..d90bfa3edd 100644 --- a/api/ateapi/v1/AGENTS.md +++ b/api/ateapi/v1/AGENTS.md @@ -3,6 +3,6 @@ `ateapi.proto` defines Substrate's public API. - See `docs/api-style-guide.md` for the API style guide. -- See `docs/resource-model.md` for the resources and subresources the API exposes. Update it when you add, remove, or re-parent a resource. +- See `docs/reference/resource-model.md` for the resources and subresources the API exposes. Update it when you add, remove, or re-parent a resource. - See `docs/api-validation.md` for how to use declarative validation (DV) tags. - `*.pb.go` files are generated. Never edit them by hand. After changing `ateapi.proto`, run `hack/update/codegen.sh`, which also regenerates the DV code in `cmd/ateapi/internal/apivalidation`. diff --git a/api/ateapi/v1/ateapi.pb.go b/api/ateapi/v1/ateapi.pb.go index 250a2f5968..44de8a372c 100644 --- a/api/ateapi/v1/ateapi.pb.go +++ b/api/ateapi/v1/ateapi.pb.go @@ -89,55 +89,55 @@ func (SnapshotOwner) EnumDescriptor() ([]byte, []int) { return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{0} } -// SnapshotStorageStatus indicates the lifecycle stage of a durable snapshot. -type SnapshotStorageStatus int32 +// SnapshotStorageState indicates the lifecycle stage of a durable snapshot. +type SnapshotStorageState int32 const ( - SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED SnapshotStorageStatus = 0 + SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED SnapshotStorageState = 0 // Snapshot creation in durable storage is in progress. - SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING SnapshotStorageStatus = 1 + SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING SnapshotStorageState = 1 // Snapshot is complete and ready for use in durable storage. - SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED SnapshotStorageStatus = 2 // Keep this in sync with SnapshotStorage.status's maximum. + SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED SnapshotStorageState = 2 // Keep this in sync with SnapshotStorage.state's maximum. ) -// Enum value maps for SnapshotStorageStatus. +// Enum value maps for SnapshotStorageState. var ( - SnapshotStorageStatus_name = map[int32]string{ - 0: "SNAPSHOT_STORAGE_STATUS_UNSPECIFIED", - 1: "SNAPSHOT_STORAGE_STATUS_PENDING", - 2: "SNAPSHOT_STORAGE_STATUS_COMPLETED", + SnapshotStorageState_name = map[int32]string{ + 0: "SNAPSHOT_STORAGE_STATE_UNSPECIFIED", + 1: "SNAPSHOT_STORAGE_STATE_PENDING", + 2: "SNAPSHOT_STORAGE_STATE_COMPLETED", } - SnapshotStorageStatus_value = map[string]int32{ - "SNAPSHOT_STORAGE_STATUS_UNSPECIFIED": 0, - "SNAPSHOT_STORAGE_STATUS_PENDING": 1, - "SNAPSHOT_STORAGE_STATUS_COMPLETED": 2, + SnapshotStorageState_value = map[string]int32{ + "SNAPSHOT_STORAGE_STATE_UNSPECIFIED": 0, + "SNAPSHOT_STORAGE_STATE_PENDING": 1, + "SNAPSHOT_STORAGE_STATE_COMPLETED": 2, } ) -func (x SnapshotStorageStatus) Enum() *SnapshotStorageStatus { - p := new(SnapshotStorageStatus) +func (x SnapshotStorageState) Enum() *SnapshotStorageState { + p := new(SnapshotStorageState) *p = x return p } -func (x SnapshotStorageStatus) String() string { +func (x SnapshotStorageState) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } -func (SnapshotStorageStatus) Descriptor() protoreflect.EnumDescriptor { +func (SnapshotStorageState) Descriptor() protoreflect.EnumDescriptor { return file_api_ateapi_v1_ateapi_proto_enumTypes[1].Descriptor() } -func (SnapshotStorageStatus) Type() protoreflect.EnumType { +func (SnapshotStorageState) Type() protoreflect.EnumType { return &file_api_ateapi_v1_ateapi_proto_enumTypes[1] } -func (x SnapshotStorageStatus) Number() protoreflect.EnumNumber { +func (x SnapshotStorageState) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } -// Deprecated: Use SnapshotStorageStatus.Descriptor instead. -func (SnapshotStorageStatus) EnumDescriptor() ([]byte, []int) { +// Deprecated: Use SnapshotStorageState.Descriptor instead. +func (SnapshotStorageState) EnumDescriptor() ([]byte, []int) { return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{1} } @@ -640,12 +640,12 @@ func (x *ObjectSnapshot) GetSnapshotUri() string { // SnapshotStorage records the durable storage representation of a Snapshot. type SnapshotStorage struct { state protoimpl.MessageState `protogen:"open.v1"` - // status is the lifecycle status of the durable snapshot. + // state is the lifecycle state of the durable snapshot. // // +k8s:required // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotStorageStatus enum - Status SnapshotStorageStatus `protobuf:"varint,1,opt,name=status,proto3,enum=substrate.ateapi.v1.SnapshotStorageStatus" json:"status,omitempty"` + // +k8s:maximum=2 # keep this in sync with the SnapshotStorageState enum + State SnapshotStorageState `protobuf:"varint,1,opt,name=state,proto3,enum=substrate.ateapi.v1.SnapshotStorageState" json:"state,omitempty"` // object snapshot is stored in object storage. // // +k8s:required @@ -684,11 +684,11 @@ func (*SnapshotStorage) Descriptor() ([]byte, []int) { return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{1} } -func (x *SnapshotStorage) GetStatus() SnapshotStorageStatus { +func (x *SnapshotStorage) GetState() SnapshotStorageState { if x != nil { - return x.Status + return x.State } - return SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED + return SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED } func (x *SnapshotStorage) GetObject() *ObjectSnapshot { @@ -3135,9 +3135,10 @@ type SnapshotConfig struct { // storage_location is the base object-storage URI snapshots of actors on // this version are stored under. If unset, ate-api-server's // --snapshot-location is used. Requires the PerTemplateSnapshotLocation preview - // gate; ignored without it. + // gate. // // +ate:preview=PerTemplateSnapshotLocation + // +k8s:ifDisabled(PerTemplateSnapshotLocation)=+k8s:forbidden // +k8s:optional // +k8s:maxLength=1024 // +k8s:customValidation # Validate URI @@ -7492,7 +7493,8 @@ type WorkerStatus struct { // +k8s:optional Capacity *WorkerResources `protobuf:"bytes,2,opt,name=capacity,proto3" json:"capacity,omitempty"` // The allocated resources of the Worker. This is the sum of all the - // resources consumed by the Actors currently assigned to this Worker. + // resources consumed by the Actors currently assigned to this Worker, + // including their runtime's actor_overhead. // // +k8s:optional Allocated *WorkerResources `protobuf:"bytes,3,opt,name=allocated,proto3" json:"allocated,omitempty"` @@ -7722,7 +7724,15 @@ type WorkerSandboxRuntime struct { // restore snapshots from. // // +k8s:required - Version *VersionedSandboxCompat `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + Version *VersionedSandboxCompat `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + // The resource overhead incurred by a Worker for each Actor running under + // this runtime, on top of the Actor's stated limits. Placement considers + // this when deciding if a Worker has enough capacity for an Actor, and it is + // accounted for in allocated. If not specified, Actors under this runtime + // are assumed to have negligible overhead. + // + // +k8s:optional + ActorOverhead *RuntimeOverhead `protobuf:"bytes,2,opt,name=actor_overhead,json=actorOverhead,proto3" json:"actor_overhead,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -7764,6 +7774,79 @@ func (x *WorkerSandboxRuntime) GetVersion() *VersionedSandboxCompat { return nil } +func (x *WorkerSandboxRuntime) GetActorOverhead() *RuntimeOverhead { + if x != nil { + return x.ActorOverhead + } + return nil +} + +// RuntimeOverhead is the memory an Actor uses on its Worker outside its own +// limits: a fixed amount, plus an amount in proportion to its memory limit. +type RuntimeOverhead struct { + state protoimpl.MessageState `protogen:"open.v1"` + // memory_bytes is a fixed amount of memory for each Actor. + // + // +k8s:optional + // +k8s:minimum=0 + // +k8s:maximum=1099511627776 # 1TiB + MemoryBytes int64 `protobuf:"varint,1,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` + // memory_bytes_per_gib is memory per GiB of the Actor's memory limit, e.g. + // host page tables for a microVM's guest RAM. It scales in proportion and + // rounds down to a byte, so an Actor with a 512MiB limit adds half of it. + // + // +k8s:optional + // +k8s:minimum=0 + // +k8s:maximum=1073741824 # 1GiB + MemoryBytesPerGib int64 `protobuf:"varint,2,opt,name=memory_bytes_per_gib,json=memoryBytesPerGib,proto3" json:"memory_bytes_per_gib,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RuntimeOverhead) Reset() { + *x = RuntimeOverhead{} + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[107] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RuntimeOverhead) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RuntimeOverhead) ProtoMessage() {} + +func (x *RuntimeOverhead) ProtoReflect() protoreflect.Message { + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[107] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RuntimeOverhead.ProtoReflect.Descriptor instead. +func (*RuntimeOverhead) Descriptor() ([]byte, []int) { + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{107} +} + +func (x *RuntimeOverhead) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + +func (x *RuntimeOverhead) GetMemoryBytesPerGib() int64 { + if x != nil { + return x.MemoryBytesPerGib + } + return 0 +} + // ActorSandboxRuntime is the runtime an actor runs under, as recorded on its // assignment and on the snapshots taken from it: one class and one of the // compatibility versions a Worker reports for it. @@ -7786,7 +7869,7 @@ type ActorSandboxRuntime struct { func (x *ActorSandboxRuntime) Reset() { *x = ActorSandboxRuntime{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[107] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[108] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7798,7 +7881,7 @@ func (x *ActorSandboxRuntime) String() string { func (*ActorSandboxRuntime) ProtoMessage() {} func (x *ActorSandboxRuntime) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[107] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[108] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7811,7 +7894,7 @@ func (x *ActorSandboxRuntime) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorSandboxRuntime.ProtoReflect.Descriptor instead. func (*ActorSandboxRuntime) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{107} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{108} } func (x *ActorSandboxRuntime) GetSandboxClass() string { @@ -7862,7 +7945,7 @@ type VersionedSandboxCompat struct { func (x *VersionedSandboxCompat) Reset() { *x = VersionedSandboxCompat{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[108] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[109] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7874,7 +7957,7 @@ func (x *VersionedSandboxCompat) String() string { func (*VersionedSandboxCompat) ProtoMessage() {} func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[108] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[109] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7887,7 +7970,7 @@ func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { // Deprecated: Use VersionedSandboxCompat.ProtoReflect.Descriptor instead. func (*VersionedSandboxCompat) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{108} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{109} } func (x *VersionedSandboxCompat) GetSchemaVersion() string { @@ -7923,7 +8006,7 @@ type AttributeEntry struct { func (x *AttributeEntry) Reset() { *x = AttributeEntry{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[109] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[110] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7935,7 +8018,7 @@ func (x *AttributeEntry) String() string { func (*AttributeEntry) ProtoMessage() {} func (x *AttributeEntry) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[109] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[110] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7948,7 +8031,7 @@ func (x *AttributeEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use AttributeEntry.ProtoReflect.Descriptor instead. func (*AttributeEntry) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{109} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{110} } func (x *AttributeEntry) GetKey() string { @@ -7995,7 +8078,7 @@ type WorkerResources struct { func (x *WorkerResources) Reset() { *x = WorkerResources{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[110] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[111] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8007,7 +8090,7 @@ func (x *WorkerResources) String() string { func (*WorkerResources) ProtoMessage() {} func (x *WorkerResources) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[110] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[111] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8020,7 +8103,7 @@ func (x *WorkerResources) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. func (*WorkerResources) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{110} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{111} } func (x *WorkerResources) GetCpuMillicores() int64 { @@ -8065,7 +8148,8 @@ type ActorAssignment struct { // +k8s:required // +k8s:format=k8s-uuid ActorUid string `protobuf:"bytes,3,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` - // The resources allocated to the Actor when it was assigned. + // The resources allocated to the Actor when it was assigned: its limits, + // plus the actor_overhead of the runtime it runs under on the Worker. // // +k8s:optional Resources *Resources `protobuf:"bytes,4,opt,name=resources,proto3" json:"resources,omitempty"` @@ -8075,7 +8159,7 @@ type ActorAssignment struct { func (x *ActorAssignment) Reset() { *x = ActorAssignment{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[111] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[112] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8087,7 +8171,7 @@ func (x *ActorAssignment) String() string { func (*ActorAssignment) ProtoMessage() {} func (x *ActorAssignment) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[111] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[112] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8100,7 +8184,7 @@ func (x *ActorAssignment) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorAssignment.ProtoReflect.Descriptor instead. func (*ActorAssignment) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{111} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{112} } func (x *ActorAssignment) GetMetadata() *ResourceMetadata { @@ -8145,7 +8229,7 @@ type AcquireWorkerEpochRequest struct { func (x *AcquireWorkerEpochRequest) Reset() { *x = AcquireWorkerEpochRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[112] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[113] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8157,7 +8241,7 @@ func (x *AcquireWorkerEpochRequest) String() string { func (*AcquireWorkerEpochRequest) ProtoMessage() {} func (x *AcquireWorkerEpochRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[112] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[113] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8170,7 +8254,7 @@ func (x *AcquireWorkerEpochRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AcquireWorkerEpochRequest.ProtoReflect.Descriptor instead. func (*AcquireWorkerEpochRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{112} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{113} } func (x *AcquireWorkerEpochRequest) GetWorker() *ObjectRef { @@ -8190,7 +8274,7 @@ type AcquireWorkerEpochResponse struct { func (x *AcquireWorkerEpochResponse) Reset() { *x = AcquireWorkerEpochResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[113] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[114] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8202,7 +8286,7 @@ func (x *AcquireWorkerEpochResponse) String() string { func (*AcquireWorkerEpochResponse) ProtoMessage() {} func (x *AcquireWorkerEpochResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[113] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[114] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8215,7 +8299,7 @@ func (x *AcquireWorkerEpochResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AcquireWorkerEpochResponse.ProtoReflect.Descriptor instead. func (*AcquireWorkerEpochResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{113} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{114} } func (x *AcquireWorkerEpochResponse) GetEpoch() int64 { @@ -8258,7 +8342,7 @@ type RegisterWorkerRequest struct { func (x *RegisterWorkerRequest) Reset() { *x = RegisterWorkerRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[114] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[115] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8270,7 +8354,7 @@ func (x *RegisterWorkerRequest) String() string { func (*RegisterWorkerRequest) ProtoMessage() {} func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[114] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[115] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8283,7 +8367,7 @@ func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerRequest.ProtoReflect.Descriptor instead. func (*RegisterWorkerRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{114} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{115} } func (x *RegisterWorkerRequest) GetWorker() *ObjectRef { @@ -8324,7 +8408,7 @@ type RegisterWorkerResponse struct { func (x *RegisterWorkerResponse) Reset() { *x = RegisterWorkerResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[115] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[116] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8336,7 +8420,7 @@ func (x *RegisterWorkerResponse) String() string { func (*RegisterWorkerResponse) ProtoMessage() {} func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[115] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[116] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8349,7 +8433,7 @@ func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerResponse.ProtoReflect.Descriptor instead. func (*RegisterWorkerResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{115} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{116} } func (x *RegisterWorkerResponse) GetWorker() *Worker { @@ -8389,7 +8473,7 @@ type SetWorkerSandboxRuntimesRequest struct { func (x *SetWorkerSandboxRuntimesRequest) Reset() { *x = SetWorkerSandboxRuntimesRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[116] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[117] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8401,7 +8485,7 @@ func (x *SetWorkerSandboxRuntimesRequest) String() string { func (*SetWorkerSandboxRuntimesRequest) ProtoMessage() {} func (x *SetWorkerSandboxRuntimesRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[116] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[117] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8414,7 +8498,7 @@ func (x *SetWorkerSandboxRuntimesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SetWorkerSandboxRuntimesRequest.ProtoReflect.Descriptor instead. func (*SetWorkerSandboxRuntimesRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{116} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{117} } func (x *SetWorkerSandboxRuntimesRequest) GetWorker() *ObjectRef { @@ -8448,7 +8532,7 @@ type SetWorkerSandboxRuntimesResponse struct { func (x *SetWorkerSandboxRuntimesResponse) Reset() { *x = SetWorkerSandboxRuntimesResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[117] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[118] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8460,7 +8544,7 @@ func (x *SetWorkerSandboxRuntimesResponse) String() string { func (*SetWorkerSandboxRuntimesResponse) ProtoMessage() {} func (x *SetWorkerSandboxRuntimesResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[117] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[118] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8473,7 +8557,7 @@ func (x *SetWorkerSandboxRuntimesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SetWorkerSandboxRuntimesResponse.ProtoReflect.Descriptor instead. func (*SetWorkerSandboxRuntimesResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{117} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{118} } func (x *SetWorkerSandboxRuntimesResponse) GetWorker() *Worker { @@ -8520,7 +8604,7 @@ type MintWorkerActorCertificateRequest struct { func (x *MintWorkerActorCertificateRequest) Reset() { *x = MintWorkerActorCertificateRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[118] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[119] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8532,7 +8616,7 @@ func (x *MintWorkerActorCertificateRequest) String() string { func (*MintWorkerActorCertificateRequest) ProtoMessage() {} func (x *MintWorkerActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[118] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[119] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8545,7 +8629,7 @@ func (x *MintWorkerActorCertificateRequest) ProtoReflect() protoreflect.Message // Deprecated: Use MintWorkerActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintWorkerActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{118} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{119} } func (x *MintWorkerActorCertificateRequest) GetActor() *ObjectRef { @@ -8589,7 +8673,7 @@ type MintWorkerActorCertificateResponse struct { func (x *MintWorkerActorCertificateResponse) Reset() { *x = MintWorkerActorCertificateResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[119] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[120] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8601,7 +8685,7 @@ func (x *MintWorkerActorCertificateResponse) String() string { func (*MintWorkerActorCertificateResponse) ProtoMessage() {} func (x *MintWorkerActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[119] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[120] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8614,7 +8698,7 @@ func (x *MintWorkerActorCertificateResponse) ProtoReflect() protoreflect.Message // Deprecated: Use MintWorkerActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintWorkerActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{119} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{120} } func (x *MintWorkerActorCertificateResponse) GetActorCertificates() [][]byte { @@ -8655,7 +8739,7 @@ type MintWorkerActorJwtRequest struct { func (x *MintWorkerActorJwtRequest) Reset() { *x = MintWorkerActorJwtRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[120] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[121] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8667,7 +8751,7 @@ func (x *MintWorkerActorJwtRequest) String() string { func (*MintWorkerActorJwtRequest) ProtoMessage() {} func (x *MintWorkerActorJwtRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[120] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[121] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8680,7 +8764,7 @@ func (x *MintWorkerActorJwtRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintWorkerActorJwtRequest.ProtoReflect.Descriptor instead. func (*MintWorkerActorJwtRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{120} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{121} } func (x *MintWorkerActorJwtRequest) GetActor() *ObjectRef { @@ -8728,7 +8812,7 @@ type MintWorkerActorJwtResponse struct { func (x *MintWorkerActorJwtResponse) Reset() { *x = MintWorkerActorJwtResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[121] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[122] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8740,7 +8824,7 @@ func (x *MintWorkerActorJwtResponse) String() string { func (*MintWorkerActorJwtResponse) ProtoMessage() {} func (x *MintWorkerActorJwtResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[121] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[122] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8753,7 +8837,7 @@ func (x *MintWorkerActorJwtResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintWorkerActorJwtResponse.ProtoReflect.Descriptor instead. func (*MintWorkerActorJwtResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{121} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{122} } func (x *MintWorkerActorJwtResponse) GetActorJwt() string { @@ -8801,7 +8885,7 @@ type RequestActorSuspendRequest struct { func (x *RequestActorSuspendRequest) Reset() { *x = RequestActorSuspendRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[122] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[123] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8813,7 +8897,7 @@ func (x *RequestActorSuspendRequest) String() string { func (*RequestActorSuspendRequest) ProtoMessage() {} func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[122] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[123] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8826,7 +8910,7 @@ func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendRequest.ProtoReflect.Descriptor instead. func (*RequestActorSuspendRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{122} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{123} } func (x *RequestActorSuspendRequest) GetWorker() *ObjectRef { @@ -8860,7 +8944,7 @@ type RequestActorSuspendResponse struct { func (x *RequestActorSuspendResponse) Reset() { *x = RequestActorSuspendResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[123] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[124] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8872,7 +8956,7 @@ func (x *RequestActorSuspendResponse) String() string { func (*RequestActorSuspendResponse) ProtoMessage() {} func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[123] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[124] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8885,7 +8969,7 @@ func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendResponse.ProtoReflect.Descriptor instead. func (*RequestActorSuspendResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{123} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{124} } func (x *RequestActorSuspendResponse) GetActor() *Actor { @@ -8925,7 +9009,7 @@ type ReportActorCrashedRequest struct { func (x *ReportActorCrashedRequest) Reset() { *x = ReportActorCrashedRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[124] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[125] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8937,7 +9021,7 @@ func (x *ReportActorCrashedRequest) String() string { func (*ReportActorCrashedRequest) ProtoMessage() {} func (x *ReportActorCrashedRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[124] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[125] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8950,7 +9034,7 @@ func (x *ReportActorCrashedRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorCrashedRequest.ProtoReflect.Descriptor instead. func (*ReportActorCrashedRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{124} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{125} } func (x *ReportActorCrashedRequest) GetWorker() *ObjectRef { @@ -8991,7 +9075,7 @@ type ReportActorCrashedResponse struct { func (x *ReportActorCrashedResponse) Reset() { *x = ReportActorCrashedResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[125] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[126] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9003,7 +9087,7 @@ func (x *ReportActorCrashedResponse) String() string { func (*ReportActorCrashedResponse) ProtoMessage() {} func (x *ReportActorCrashedResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[125] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[126] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9016,7 +9100,7 @@ func (x *ReportActorCrashedResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorCrashedResponse.ProtoReflect.Descriptor instead. func (*ReportActorCrashedResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{125} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{126} } func (x *ReportActorCrashedResponse) GetActor() *Actor { @@ -9047,7 +9131,7 @@ type ReportActorSnapshotStateRequest struct { func (x *ReportActorSnapshotStateRequest) Reset() { *x = ReportActorSnapshotStateRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[126] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[127] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9059,7 +9143,7 @@ func (x *ReportActorSnapshotStateRequest) String() string { func (*ReportActorSnapshotStateRequest) ProtoMessage() {} func (x *ReportActorSnapshotStateRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[126] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[127] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9072,7 +9156,7 @@ func (x *ReportActorSnapshotStateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorSnapshotStateRequest.ProtoReflect.Descriptor instead. func (*ReportActorSnapshotStateRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{126} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{127} } func (x *ReportActorSnapshotStateRequest) GetActor() *ObjectRef { @@ -9104,7 +9188,7 @@ type ReportActorSnapshotStateResponse struct { func (x *ReportActorSnapshotStateResponse) Reset() { *x = ReportActorSnapshotStateResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[127] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[128] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9116,7 +9200,7 @@ func (x *ReportActorSnapshotStateResponse) String() string { func (*ReportActorSnapshotStateResponse) ProtoMessage() {} func (x *ReportActorSnapshotStateResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[127] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[128] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9129,7 +9213,7 @@ func (x *ReportActorSnapshotStateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorSnapshotStateResponse.ProtoReflect.Descriptor instead. func (*ReportActorSnapshotStateResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{127} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{128} } // ActorSnapshotState is what atelet holds for one Actor on its node. It @@ -9163,7 +9247,7 @@ type ActorSnapshotState struct { func (x *ActorSnapshotState) Reset() { *x = ActorSnapshotState{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[128] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[129] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9175,7 +9259,7 @@ func (x *ActorSnapshotState) String() string { func (*ActorSnapshotState) ProtoMessage() {} func (x *ActorSnapshotState) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[128] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[129] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9188,7 +9272,7 @@ func (x *ActorSnapshotState) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorSnapshotState.ProtoReflect.Descriptor instead. func (*ActorSnapshotState) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{128} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{129} } func (x *ActorSnapshotState) GetLatestLocal() *SnapshotReport { @@ -9260,7 +9344,7 @@ type SnapshotReport struct { func (x *SnapshotReport) Reset() { *x = SnapshotReport{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[129] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[130] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9272,7 +9356,7 @@ func (x *SnapshotReport) String() string { func (*SnapshotReport) ProtoMessage() {} func (x *SnapshotReport) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[129] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[130] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9285,7 +9369,7 @@ func (x *SnapshotReport) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotReport.ProtoReflect.Descriptor instead. func (*SnapshotReport) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{129} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{130} } func (x *SnapshotReport) GetUuid() string { @@ -9355,7 +9439,7 @@ type AccessPolicy struct { func (x *AccessPolicy) Reset() { *x = AccessPolicy{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[130] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[131] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9367,7 +9451,7 @@ func (x *AccessPolicy) String() string { func (*AccessPolicy) ProtoMessage() {} func (x *AccessPolicy) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[130] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[131] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9380,7 +9464,7 @@ func (x *AccessPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use AccessPolicy.ProtoReflect.Descriptor instead. func (*AccessPolicy) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{130} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{131} } func (x *AccessPolicy) GetMetadata() *ResourceMetadata { @@ -9422,7 +9506,7 @@ type Binding struct { func (x *Binding) Reset() { *x = Binding{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[131] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[132] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9434,7 +9518,7 @@ func (x *Binding) String() string { func (*Binding) ProtoMessage() {} func (x *Binding) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[131] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[132] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9447,7 +9531,7 @@ func (x *Binding) ProtoReflect() protoreflect.Message { // Deprecated: Use Binding.ProtoReflect.Descriptor instead. func (*Binding) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{131} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{132} } func (x *Binding) GetRole() string { @@ -9479,7 +9563,7 @@ type GetGlobalAccessPolicyRequest struct { func (x *GetGlobalAccessPolicyRequest) Reset() { *x = GetGlobalAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[132] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[133] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9491,7 +9575,7 @@ func (x *GetGlobalAccessPolicyRequest) String() string { func (*GetGlobalAccessPolicyRequest) ProtoMessage() {} func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[132] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[133] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9504,7 +9588,7 @@ func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{132} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{133} } func (x *GetGlobalAccessPolicyRequest) GetName() string { @@ -9528,7 +9612,7 @@ type CreateGlobalAccessPolicyRequest struct { func (x *CreateGlobalAccessPolicyRequest) Reset() { *x = CreateGlobalAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[133] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[134] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9540,7 +9624,7 @@ func (x *CreateGlobalAccessPolicyRequest) String() string { func (*CreateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[133] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[134] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9553,7 +9637,7 @@ func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{133} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{134} } func (x *CreateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -9577,7 +9661,7 @@ type UpdateGlobalAccessPolicyRequest struct { func (x *UpdateGlobalAccessPolicyRequest) Reset() { *x = UpdateGlobalAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[134] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[135] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9589,7 +9673,7 @@ func (x *UpdateGlobalAccessPolicyRequest) String() string { func (*UpdateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[134] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[135] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9602,7 +9686,7 @@ func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{134} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{135} } func (x *UpdateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -9632,7 +9716,7 @@ type GetAtespaceAccessPolicyRequest struct { func (x *GetAtespaceAccessPolicyRequest) Reset() { *x = GetAtespaceAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[135] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[136] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9644,7 +9728,7 @@ func (x *GetAtespaceAccessPolicyRequest) String() string { func (*GetAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[135] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[136] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9657,7 +9741,7 @@ func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{135} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{136} } func (x *GetAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9693,7 +9777,7 @@ type CreateAtespaceAccessPolicyRequest struct { func (x *CreateAtespaceAccessPolicyRequest) Reset() { *x = CreateAtespaceAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[136] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[137] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9705,7 +9789,7 @@ func (x *CreateAtespaceAccessPolicyRequest) String() string { func (*CreateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[136] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[137] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9718,7 +9802,7 @@ func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use CreateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{136} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{137} } func (x *CreateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9754,7 +9838,7 @@ type UpdateAtespaceAccessPolicyRequest struct { func (x *UpdateAtespaceAccessPolicyRequest) Reset() { *x = UpdateAtespaceAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[137] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[138] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9766,7 +9850,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) String() string { func (*UpdateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[137] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[138] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9779,7 +9863,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use UpdateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{137} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{138} } func (x *UpdateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9820,7 +9904,7 @@ type DeleteAtespaceAccessPolicyRequest struct { func (x *DeleteAtespaceAccessPolicyRequest) Reset() { *x = DeleteAtespaceAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[138] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[139] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9832,7 +9916,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) String() string { func (*DeleteAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[138] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[139] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9845,7 +9929,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use DeleteAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*DeleteAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{138} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{139} } func (x *DeleteAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9891,7 +9975,7 @@ type ResourceAnnotation struct { func (x *ResourceAnnotation) Reset() { *x = ResourceAnnotation{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[139] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[140] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9903,7 +9987,7 @@ func (x *ResourceAnnotation) String() string { func (*ResourceAnnotation) ProtoMessage() {} func (x *ResourceAnnotation) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[139] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[140] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9916,7 +10000,7 @@ func (x *ResourceAnnotation) ProtoReflect() protoreflect.Message { // Deprecated: Use ResourceAnnotation.ProtoReflect.Descriptor instead. func (*ResourceAnnotation) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{139} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{140} } func (x *ResourceAnnotation) GetParents() []string { @@ -9959,7 +10043,7 @@ type MethodAnnotation struct { func (x *MethodAnnotation) Reset() { *x = MethodAnnotation{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[140] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[141] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9971,7 +10055,7 @@ func (x *MethodAnnotation) String() string { func (*MethodAnnotation) ProtoMessage() {} func (x *MethodAnnotation) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[140] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[141] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9984,7 +10068,7 @@ func (x *MethodAnnotation) ProtoReflect() protoreflect.Message { // Deprecated: Use MethodAnnotation.ProtoReflect.Descriptor instead. func (*MethodAnnotation) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{140} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{141} } func (x *MethodAnnotation) GetResource() string { @@ -10031,9 +10115,9 @@ const file_api_ateapi_v1_ateapi_proto_rawDesc = "" + "\n" + "\x1aapi/ateapi/v1/ateapi.proto\x12\x13substrate.ateapi.v1\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"3\n" + "\x0eObjectSnapshot\x12!\n" + - "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"\x92\x01\n" + - "\x0fSnapshotStorage\x12B\n" + - "\x06status\x18\x01 \x01(\x0e2*.substrate.ateapi.v1.SnapshotStorageStatusR\x06status\x12;\n" + + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"\x8f\x01\n" + + "\x0fSnapshotStorage\x12?\n" + + "\x05state\x18\x01 \x01(\x0e2).substrate.ateapi.v1.SnapshotStorageStateR\x05state\x12;\n" + "\x06object\x18\x02 \x01(\v2#.substrate.ateapi.v1.ObjectSnapshotR\x06object\"\xa9\x03\n" + "\bSnapshot\x12\x12\n" + "\x04uuid\x18\x01 \x01(\tR\x04uuid\x12\x1e\n" + @@ -10408,9 +10492,13 @@ const file_api_ateapi_v1_ateapi_proto_rawDesc = "" + "\x12WorkerSandboxClass\x12#\n" + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x12R\n" + "\x0fdefault_runtime\x18\x02 \x01(\v2).substrate.ateapi.v1.WorkerSandboxRuntimeR\x0edefaultRuntime\x12Z\n" + - "\x13restorable_runtimes\x18\x03 \x03(\v2).substrate.ateapi.v1.WorkerSandboxRuntimeR\x12restorableRuntimes\"]\n" + + "\x13restorable_runtimes\x18\x03 \x03(\v2).substrate.ateapi.v1.WorkerSandboxRuntimeR\x12restorableRuntimes\"\xaa\x01\n" + "\x14WorkerSandboxRuntime\x12E\n" + - "\aversion\x18\x01 \x01(\v2+.substrate.ateapi.v1.VersionedSandboxCompatR\aversion\"\x81\x01\n" + + "\aversion\x18\x01 \x01(\v2+.substrate.ateapi.v1.VersionedSandboxCompatR\aversion\x12K\n" + + "\x0eactor_overhead\x18\x02 \x01(\v2$.substrate.ateapi.v1.RuntimeOverheadR\ractorOverhead\"e\n" + + "\x0fRuntimeOverhead\x12!\n" + + "\fmemory_bytes\x18\x01 \x01(\x03R\vmemoryBytes\x12/\n" + + "\x14memory_bytes_per_gib\x18\x02 \x01(\x03R\x11memoryBytesPerGib\"\x81\x01\n" + "\x13ActorSandboxRuntime\x12#\n" + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x12E\n" + "\aversion\x18\x02 \x01(\v2+.substrate.ateapi.v1.VersionedSandboxCompatR\aversion\"\x84\x01\n" + @@ -10533,11 +10621,11 @@ const file_api_ateapi_v1_ateapi_proto_rawDesc = "" + "\rSnapshotOwner\x12\x1e\n" + "\x1aSNAPSHOT_OWNER_UNSPECIFIED\x10\x00\x12\x18\n" + "\x14SNAPSHOT_OWNER_ACTOR\x10\x01\x12\x16\n" + - "\x12SNAPSHOT_OWNER_TAG\x10\x02*\x8c\x01\n" + - "\x15SnapshotStorageStatus\x12'\n" + - "#SNAPSHOT_STORAGE_STATUS_UNSPECIFIED\x10\x00\x12#\n" + - "\x1fSNAPSHOT_STORAGE_STATUS_PENDING\x10\x01\x12%\n" + - "!SNAPSHOT_STORAGE_STATUS_COMPLETED\x10\x02*\x90\x01\n" + + "\x12SNAPSHOT_OWNER_TAG\x10\x02*\x88\x01\n" + + "\x14SnapshotStorageState\x12&\n" + + "\"SNAPSHOT_STORAGE_STATE_UNSPECIFIED\x10\x00\x12\"\n" + + "\x1eSNAPSHOT_STORAGE_STATE_PENDING\x10\x01\x12$\n" + + " SNAPSHOT_STORAGE_STATE_COMPLETED\x10\x02*\x90\x01\n" + "\x10SnapshotFidelity\x12!\n" + "\x1dSNAPSHOT_FIDELITY_UNSPECIFIED\x10\x00\x12\x1d\n" + "\x19SNAPSHOT_FIDELITY_VOLUMES\x10\x01\x12\x1c\n" + @@ -10696,10 +10784,10 @@ func file_api_ateapi_v1_ateapi_proto_rawDescGZIP() []byte { } var file_api_ateapi_v1_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 10) -var file_api_ateapi_v1_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 144) +var file_api_ateapi_v1_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 145) var file_api_ateapi_v1_ateapi_proto_goTypes = []any{ (SnapshotOwner)(0), // 0: substrate.ateapi.v1.SnapshotOwner - (SnapshotStorageStatus)(0), // 1: substrate.ateapi.v1.SnapshotStorageStatus + (SnapshotStorageState)(0), // 1: substrate.ateapi.v1.SnapshotStorageState (SnapshotFidelity)(0), // 2: substrate.ateapi.v1.SnapshotFidelity (TagScope)(0), // 3: substrate.ateapi.v1.TagScope (ActorState)(0), // 4: substrate.ateapi.v1.ActorState @@ -10815,60 +10903,61 @@ var file_api_ateapi_v1_ateapi_proto_goTypes = []any{ (*WorkerStatus)(nil), // 114: substrate.ateapi.v1.WorkerStatus (*WorkerSandboxClass)(nil), // 115: substrate.ateapi.v1.WorkerSandboxClass (*WorkerSandboxRuntime)(nil), // 116: substrate.ateapi.v1.WorkerSandboxRuntime - (*ActorSandboxRuntime)(nil), // 117: substrate.ateapi.v1.ActorSandboxRuntime - (*VersionedSandboxCompat)(nil), // 118: substrate.ateapi.v1.VersionedSandboxCompat - (*AttributeEntry)(nil), // 119: substrate.ateapi.v1.AttributeEntry - (*WorkerResources)(nil), // 120: substrate.ateapi.v1.WorkerResources - (*ActorAssignment)(nil), // 121: substrate.ateapi.v1.ActorAssignment - (*AcquireWorkerEpochRequest)(nil), // 122: substrate.ateapi.v1.AcquireWorkerEpochRequest - (*AcquireWorkerEpochResponse)(nil), // 123: substrate.ateapi.v1.AcquireWorkerEpochResponse - (*RegisterWorkerRequest)(nil), // 124: substrate.ateapi.v1.RegisterWorkerRequest - (*RegisterWorkerResponse)(nil), // 125: substrate.ateapi.v1.RegisterWorkerResponse - (*SetWorkerSandboxRuntimesRequest)(nil), // 126: substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest - (*SetWorkerSandboxRuntimesResponse)(nil), // 127: substrate.ateapi.v1.SetWorkerSandboxRuntimesResponse - (*MintWorkerActorCertificateRequest)(nil), // 128: substrate.ateapi.v1.MintWorkerActorCertificateRequest - (*MintWorkerActorCertificateResponse)(nil), // 129: substrate.ateapi.v1.MintWorkerActorCertificateResponse - (*MintWorkerActorJwtRequest)(nil), // 130: substrate.ateapi.v1.MintWorkerActorJwtRequest - (*MintWorkerActorJwtResponse)(nil), // 131: substrate.ateapi.v1.MintWorkerActorJwtResponse - (*RequestActorSuspendRequest)(nil), // 132: substrate.ateapi.v1.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 133: substrate.ateapi.v1.RequestActorSuspendResponse - (*ReportActorCrashedRequest)(nil), // 134: substrate.ateapi.v1.ReportActorCrashedRequest - (*ReportActorCrashedResponse)(nil), // 135: substrate.ateapi.v1.ReportActorCrashedResponse - (*ReportActorSnapshotStateRequest)(nil), // 136: substrate.ateapi.v1.ReportActorSnapshotStateRequest - (*ReportActorSnapshotStateResponse)(nil), // 137: substrate.ateapi.v1.ReportActorSnapshotStateResponse - (*ActorSnapshotState)(nil), // 138: substrate.ateapi.v1.ActorSnapshotState - (*SnapshotReport)(nil), // 139: substrate.ateapi.v1.SnapshotReport - (*AccessPolicy)(nil), // 140: substrate.ateapi.v1.AccessPolicy - (*Binding)(nil), // 141: substrate.ateapi.v1.Binding - (*GetGlobalAccessPolicyRequest)(nil), // 142: substrate.ateapi.v1.GetGlobalAccessPolicyRequest - (*CreateGlobalAccessPolicyRequest)(nil), // 143: substrate.ateapi.v1.CreateGlobalAccessPolicyRequest - (*UpdateGlobalAccessPolicyRequest)(nil), // 144: substrate.ateapi.v1.UpdateGlobalAccessPolicyRequest - (*GetAtespaceAccessPolicyRequest)(nil), // 145: substrate.ateapi.v1.GetAtespaceAccessPolicyRequest - (*CreateAtespaceAccessPolicyRequest)(nil), // 146: substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest - (*UpdateAtespaceAccessPolicyRequest)(nil), // 147: substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest - (*DeleteAtespaceAccessPolicyRequest)(nil), // 148: substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest - (*ResourceAnnotation)(nil), // 149: substrate.ateapi.v1.ResourceAnnotation - (*MethodAnnotation)(nil), // 150: substrate.ateapi.v1.MethodAnnotation - nil, // 151: substrate.ateapi.v1.Selector.MatchLabelsEntry - nil, // 152: substrate.ateapi.v1.ExternalVolumeStatus.VolumeContextEntry - nil, // 153: substrate.ateapi.v1.Worker.LabelsEntry - (*timestamppb.Timestamp)(nil), // 154: google.protobuf.Timestamp - (*descriptorpb.MessageOptions)(nil), // 155: google.protobuf.MessageOptions - (*descriptorpb.MethodOptions)(nil), // 156: google.protobuf.MethodOptions + (*RuntimeOverhead)(nil), // 117: substrate.ateapi.v1.RuntimeOverhead + (*ActorSandboxRuntime)(nil), // 118: substrate.ateapi.v1.ActorSandboxRuntime + (*VersionedSandboxCompat)(nil), // 119: substrate.ateapi.v1.VersionedSandboxCompat + (*AttributeEntry)(nil), // 120: substrate.ateapi.v1.AttributeEntry + (*WorkerResources)(nil), // 121: substrate.ateapi.v1.WorkerResources + (*ActorAssignment)(nil), // 122: substrate.ateapi.v1.ActorAssignment + (*AcquireWorkerEpochRequest)(nil), // 123: substrate.ateapi.v1.AcquireWorkerEpochRequest + (*AcquireWorkerEpochResponse)(nil), // 124: substrate.ateapi.v1.AcquireWorkerEpochResponse + (*RegisterWorkerRequest)(nil), // 125: substrate.ateapi.v1.RegisterWorkerRequest + (*RegisterWorkerResponse)(nil), // 126: substrate.ateapi.v1.RegisterWorkerResponse + (*SetWorkerSandboxRuntimesRequest)(nil), // 127: substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest + (*SetWorkerSandboxRuntimesResponse)(nil), // 128: substrate.ateapi.v1.SetWorkerSandboxRuntimesResponse + (*MintWorkerActorCertificateRequest)(nil), // 129: substrate.ateapi.v1.MintWorkerActorCertificateRequest + (*MintWorkerActorCertificateResponse)(nil), // 130: substrate.ateapi.v1.MintWorkerActorCertificateResponse + (*MintWorkerActorJwtRequest)(nil), // 131: substrate.ateapi.v1.MintWorkerActorJwtRequest + (*MintWorkerActorJwtResponse)(nil), // 132: substrate.ateapi.v1.MintWorkerActorJwtResponse + (*RequestActorSuspendRequest)(nil), // 133: substrate.ateapi.v1.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 134: substrate.ateapi.v1.RequestActorSuspendResponse + (*ReportActorCrashedRequest)(nil), // 135: substrate.ateapi.v1.ReportActorCrashedRequest + (*ReportActorCrashedResponse)(nil), // 136: substrate.ateapi.v1.ReportActorCrashedResponse + (*ReportActorSnapshotStateRequest)(nil), // 137: substrate.ateapi.v1.ReportActorSnapshotStateRequest + (*ReportActorSnapshotStateResponse)(nil), // 138: substrate.ateapi.v1.ReportActorSnapshotStateResponse + (*ActorSnapshotState)(nil), // 139: substrate.ateapi.v1.ActorSnapshotState + (*SnapshotReport)(nil), // 140: substrate.ateapi.v1.SnapshotReport + (*AccessPolicy)(nil), // 141: substrate.ateapi.v1.AccessPolicy + (*Binding)(nil), // 142: substrate.ateapi.v1.Binding + (*GetGlobalAccessPolicyRequest)(nil), // 143: substrate.ateapi.v1.GetGlobalAccessPolicyRequest + (*CreateGlobalAccessPolicyRequest)(nil), // 144: substrate.ateapi.v1.CreateGlobalAccessPolicyRequest + (*UpdateGlobalAccessPolicyRequest)(nil), // 145: substrate.ateapi.v1.UpdateGlobalAccessPolicyRequest + (*GetAtespaceAccessPolicyRequest)(nil), // 146: substrate.ateapi.v1.GetAtespaceAccessPolicyRequest + (*CreateAtespaceAccessPolicyRequest)(nil), // 147: substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest + (*UpdateAtespaceAccessPolicyRequest)(nil), // 148: substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest + (*DeleteAtespaceAccessPolicyRequest)(nil), // 149: substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest + (*ResourceAnnotation)(nil), // 150: substrate.ateapi.v1.ResourceAnnotation + (*MethodAnnotation)(nil), // 151: substrate.ateapi.v1.MethodAnnotation + nil, // 152: substrate.ateapi.v1.Selector.MatchLabelsEntry + nil, // 153: substrate.ateapi.v1.ExternalVolumeStatus.VolumeContextEntry + nil, // 154: substrate.ateapi.v1.Worker.LabelsEntry + (*timestamppb.Timestamp)(nil), // 155: google.protobuf.Timestamp + (*descriptorpb.MessageOptions)(nil), // 156: google.protobuf.MessageOptions + (*descriptorpb.MethodOptions)(nil), // 157: google.protobuf.MethodOptions } var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ - 1, // 0: substrate.ateapi.v1.SnapshotStorage.status:type_name -> substrate.ateapi.v1.SnapshotStorageStatus + 1, // 0: substrate.ateapi.v1.SnapshotStorage.state:type_name -> substrate.ateapi.v1.SnapshotStorageState 10, // 1: substrate.ateapi.v1.SnapshotStorage.object:type_name -> substrate.ateapi.v1.ObjectSnapshot 0, // 2: substrate.ateapi.v1.Snapshot.owner:type_name -> substrate.ateapi.v1.SnapshotOwner 2, // 3: substrate.ateapi.v1.Snapshot.fidelity:type_name -> substrate.ateapi.v1.SnapshotFidelity 11, // 4: substrate.ateapi.v1.Snapshot.durable_snapshot:type_name -> substrate.ateapi.v1.SnapshotStorage - 117, // 5: substrate.ateapi.v1.Snapshot.sandbox_runtime:type_name -> substrate.ateapi.v1.ActorSandboxRuntime - 151, // 6: substrate.ateapi.v1.Selector.match_labels:type_name -> substrate.ateapi.v1.Selector.MatchLabelsEntry - 154, // 7: substrate.ateapi.v1.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp - 154, // 8: substrate.ateapi.v1.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp - 154, // 9: substrate.ateapi.v1.ResourceMetadata.delete_time:type_name -> google.protobuf.Timestamp + 118, // 5: substrate.ateapi.v1.Snapshot.sandbox_runtime:type_name -> substrate.ateapi.v1.ActorSandboxRuntime + 152, // 6: substrate.ateapi.v1.Selector.match_labels:type_name -> substrate.ateapi.v1.Selector.MatchLabelsEntry + 155, // 7: substrate.ateapi.v1.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 155, // 8: substrate.ateapi.v1.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 155, // 9: substrate.ateapi.v1.ResourceMetadata.delete_time:type_name -> google.protobuf.Timestamp 16, // 10: substrate.ateapi.v1.ExternalVolumes.volumes:type_name -> substrate.ateapi.v1.ExternalVolumeStatus - 152, // 11: substrate.ateapi.v1.ExternalVolumeStatus.volume_context:type_name -> substrate.ateapi.v1.ExternalVolumeStatus.VolumeContextEntry + 153, // 11: substrate.ateapi.v1.ExternalVolumeStatus.volume_context:type_name -> substrate.ateapi.v1.ExternalVolumeStatus.VolumeContextEntry 14, // 12: substrate.ateapi.v1.Actor.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata 35, // 13: substrate.ateapi.v1.Actor.actor_template:type_name -> substrate.ateapi.v1.ObjectRef 13, // 14: substrate.ateapi.v1.Actor.worker_selector:type_name -> substrate.ateapi.v1.Selector @@ -10893,9 +10982,9 @@ var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ 15, // 33: substrate.ateapi.v1.ActorStatus.external_volumes:type_name -> substrate.ateapi.v1.ExternalVolumes 30, // 34: substrate.ateapi.v1.ActorStatus.crash:type_name -> substrate.ateapi.v1.ActorCrash 12, // 35: substrate.ateapi.v1.ActorStatus.snapshots:type_name -> substrate.ateapi.v1.Snapshot - 154, // 36: substrate.ateapi.v1.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp + 155, // 36: substrate.ateapi.v1.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp 35, // 37: substrate.ateapi.v1.WorkerAssignment.worker:type_name -> substrate.ateapi.v1.ObjectRef - 117, // 38: substrate.ateapi.v1.WorkerAssignment.sandbox_runtime:type_name -> substrate.ateapi.v1.ActorSandboxRuntime + 118, // 38: substrate.ateapi.v1.WorkerAssignment.sandbox_runtime:type_name -> substrate.ateapi.v1.ActorSandboxRuntime 12, // 39: substrate.ateapi.v1.TagStatus.snapshot:type_name -> substrate.ateapi.v1.Snapshot 14, // 40: substrate.ateapi.v1.Tag.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata 32, // 41: substrate.ateapi.v1.Tag.status:type_name -> substrate.ateapi.v1.TagStatus @@ -10912,7 +11001,7 @@ var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ 40, // 52: substrate.ateapi.v1.ActorTemplate.status:type_name -> substrate.ateapi.v1.ActorTemplateStatus 38, // 53: substrate.ateapi.v1.ResourceRequirements.limits:type_name -> substrate.ateapi.v1.Resources 35, // 54: substrate.ateapi.v1.GoldenSnapshotStatus.golden_tag:type_name -> substrate.ateapi.v1.ObjectRef - 154, // 55: substrate.ateapi.v1.GoldenSnapshotStatus.snapshot_time:type_name -> google.protobuf.Timestamp + 155, // 55: substrate.ateapi.v1.GoldenSnapshotStatus.snapshot_time:type_name -> google.protobuf.Timestamp 39, // 56: substrate.ateapi.v1.ActorTemplateStatus.golden_snapshot_status:type_name -> substrate.ateapi.v1.GoldenSnapshotStatus 2, // 57: substrate.ateapi.v1.SnapshotConfig.preferred_fidelity:type_name -> substrate.ateapi.v1.SnapshotFidelity 42, // 58: substrate.ateapi.v1.SnapshotConfig.golden_snapshot_config:type_name -> substrate.ateapi.v1.GoldenSnapshotConfig @@ -10968,7 +11057,7 @@ var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ 95, // 108: substrate.ateapi.v1.DeleteActorEgressPolicyRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions 35, // 109: substrate.ateapi.v1.GetTagRequest.tag:type_name -> substrate.ateapi.v1.ObjectRef 35, // 110: substrate.ateapi.v1.MintActorJwtRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef - 154, // 111: substrate.ateapi.v1.MintActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp + 155, // 111: substrate.ateapi.v1.MintActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp 35, // 112: substrate.ateapi.v1.MintActorCertificateRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef 96, // 113: substrate.ateapi.v1.ListTagsRequest.options:type_name -> substrate.ateapi.v1.ListOptions 33, // 114: substrate.ateapi.v1.ListTagsResponse.tags:type_name -> substrate.ateapi.v1.Tag @@ -10978,7 +11067,7 @@ var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ 95, // 118: substrate.ateapi.v1.DeleteTagRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions 35, // 119: substrate.ateapi.v1.ListWorkerActorAssignmentsRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef 96, // 120: substrate.ateapi.v1.ListWorkerActorAssignmentsRequest.options:type_name -> substrate.ateapi.v1.ListOptions - 121, // 121: substrate.ateapi.v1.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> substrate.ateapi.v1.ActorAssignment + 122, // 121: substrate.ateapi.v1.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> substrate.ateapi.v1.ActorAssignment 96, // 122: substrate.ateapi.v1.ListWorkersRequest.options:type_name -> substrate.ateapi.v1.ListOptions 113, // 123: substrate.ateapi.v1.ListWorkersResponse.workers:type_name -> substrate.ateapi.v1.Worker 35, // 124: substrate.ateapi.v1.GetWorkerRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef @@ -10995,170 +11084,171 @@ var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ 96, // 135: substrate.ateapi.v1.ListActorsRequest.options:type_name -> substrate.ateapi.v1.ListOptions 17, // 136: substrate.ateapi.v1.ListActorsResponse.actors:type_name -> substrate.ateapi.v1.Actor 14, // 137: substrate.ateapi.v1.Worker.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata - 153, // 138: substrate.ateapi.v1.Worker.labels:type_name -> substrate.ateapi.v1.Worker.LabelsEntry + 154, // 138: substrate.ateapi.v1.Worker.labels:type_name -> substrate.ateapi.v1.Worker.LabelsEntry 114, // 139: substrate.ateapi.v1.Worker.status:type_name -> substrate.ateapi.v1.WorkerStatus 8, // 140: substrate.ateapi.v1.WorkerStatus.state:type_name -> substrate.ateapi.v1.WorkerState - 120, // 141: substrate.ateapi.v1.WorkerStatus.capacity:type_name -> substrate.ateapi.v1.WorkerResources - 120, // 142: substrate.ateapi.v1.WorkerStatus.allocated:type_name -> substrate.ateapi.v1.WorkerResources + 121, // 141: substrate.ateapi.v1.WorkerStatus.capacity:type_name -> substrate.ateapi.v1.WorkerResources + 121, // 142: substrate.ateapi.v1.WorkerStatus.allocated:type_name -> substrate.ateapi.v1.WorkerResources 115, // 143: substrate.ateapi.v1.WorkerStatus.sandbox_classes:type_name -> substrate.ateapi.v1.WorkerSandboxClass 116, // 144: substrate.ateapi.v1.WorkerSandboxClass.default_runtime:type_name -> substrate.ateapi.v1.WorkerSandboxRuntime 116, // 145: substrate.ateapi.v1.WorkerSandboxClass.restorable_runtimes:type_name -> substrate.ateapi.v1.WorkerSandboxRuntime - 118, // 146: substrate.ateapi.v1.WorkerSandboxRuntime.version:type_name -> substrate.ateapi.v1.VersionedSandboxCompat - 118, // 147: substrate.ateapi.v1.ActorSandboxRuntime.version:type_name -> substrate.ateapi.v1.VersionedSandboxCompat - 119, // 148: substrate.ateapi.v1.VersionedSandboxCompat.attributes:type_name -> substrate.ateapi.v1.AttributeEntry - 14, // 149: substrate.ateapi.v1.ActorAssignment.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata - 35, // 150: substrate.ateapi.v1.ActorAssignment.actor:type_name -> substrate.ateapi.v1.ObjectRef - 38, // 151: substrate.ateapi.v1.ActorAssignment.resources:type_name -> substrate.ateapi.v1.Resources - 35, // 152: substrate.ateapi.v1.AcquireWorkerEpochRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef - 35, // 153: substrate.ateapi.v1.RegisterWorkerRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef - 120, // 154: substrate.ateapi.v1.RegisterWorkerRequest.capacity:type_name -> substrate.ateapi.v1.WorkerResources - 113, // 155: substrate.ateapi.v1.RegisterWorkerResponse.worker:type_name -> substrate.ateapi.v1.Worker - 35, // 156: substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef - 115, // 157: substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest.sandbox_classes:type_name -> substrate.ateapi.v1.WorkerSandboxClass - 113, // 158: substrate.ateapi.v1.SetWorkerSandboxRuntimesResponse.worker:type_name -> substrate.ateapi.v1.Worker - 35, // 159: substrate.ateapi.v1.MintWorkerActorCertificateRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef - 35, // 160: substrate.ateapi.v1.MintWorkerActorCertificateRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef - 35, // 161: substrate.ateapi.v1.MintWorkerActorJwtRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef - 154, // 162: substrate.ateapi.v1.MintWorkerActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp - 35, // 163: substrate.ateapi.v1.RequestActorSuspendRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef - 35, // 164: substrate.ateapi.v1.RequestActorSuspendRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef - 17, // 165: substrate.ateapi.v1.RequestActorSuspendResponse.actor:type_name -> substrate.ateapi.v1.Actor - 35, // 166: substrate.ateapi.v1.ReportActorCrashedRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef - 35, // 167: substrate.ateapi.v1.ReportActorCrashedRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef - 17, // 168: substrate.ateapi.v1.ReportActorCrashedResponse.actor:type_name -> substrate.ateapi.v1.Actor - 35, // 169: substrate.ateapi.v1.ReportActorSnapshotStateRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef - 138, // 170: substrate.ateapi.v1.ReportActorSnapshotStateRequest.state:type_name -> substrate.ateapi.v1.ActorSnapshotState - 139, // 171: substrate.ateapi.v1.ActorSnapshotState.latest_local:type_name -> substrate.ateapi.v1.SnapshotReport - 139, // 172: substrate.ateapi.v1.ActorSnapshotState.uploading:type_name -> substrate.ateapi.v1.SnapshotReport - 139, // 173: substrate.ateapi.v1.ActorSnapshotState.latest_durable:type_name -> substrate.ateapi.v1.SnapshotReport - 154, // 174: substrate.ateapi.v1.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp - 2, // 175: substrate.ateapi.v1.SnapshotReport.fidelity:type_name -> substrate.ateapi.v1.SnapshotFidelity - 10, // 176: substrate.ateapi.v1.SnapshotReport.object:type_name -> substrate.ateapi.v1.ObjectSnapshot - 154, // 177: substrate.ateapi.v1.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp - 154, // 178: substrate.ateapi.v1.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp - 14, // 179: substrate.ateapi.v1.AccessPolicy.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata - 141, // 180: substrate.ateapi.v1.AccessPolicy.bindings:type_name -> substrate.ateapi.v1.Binding - 140, // 181: substrate.ateapi.v1.CreateGlobalAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy - 140, // 182: substrate.ateapi.v1.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy - 35, // 183: substrate.ateapi.v1.GetAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef - 35, // 184: substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef - 140, // 185: substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy - 35, // 186: substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef - 140, // 187: substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy - 35, // 188: substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef - 95, // 189: substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions - 9, // 190: substrate.ateapi.v1.ResourceAnnotation.scope:type_name -> substrate.ateapi.v1.Scope - 155, // 191: substrate.ateapi.v1.resource:extendee -> google.protobuf.MessageOptions - 156, // 192: substrate.ateapi.v1.method:extendee -> google.protobuf.MethodOptions - 149, // 193: substrate.ateapi.v1.resource:type_name -> substrate.ateapi.v1.ResourceAnnotation - 150, // 194: substrate.ateapi.v1.method:type_name -> substrate.ateapi.v1.MethodAnnotation - 70, // 195: substrate.ateapi.v1.Control.GetActor:input_type -> substrate.ateapi.v1.GetActorRequest - 71, // 196: substrate.ateapi.v1.Control.CreateActor:input_type -> substrate.ateapi.v1.CreateActorRequest - 72, // 197: substrate.ateapi.v1.Control.UpdateActor:input_type -> substrate.ateapi.v1.UpdateActorRequest - 73, // 198: substrate.ateapi.v1.Control.SuspendActor:input_type -> substrate.ateapi.v1.SuspendActorRequest - 76, // 199: substrate.ateapi.v1.Control.ResumeActor:input_type -> substrate.ateapi.v1.ResumeActorRequest - 78, // 200: substrate.ateapi.v1.Control.RevertActor:input_type -> substrate.ateapi.v1.RevertActorRequest - 80, // 201: substrate.ateapi.v1.Control.DeleteActor:input_type -> substrate.ateapi.v1.DeleteActorRequest - 81, // 202: substrate.ateapi.v1.Control.GetActorEgressPolicy:input_type -> substrate.ateapi.v1.GetActorEgressPolicyRequest - 82, // 203: substrate.ateapi.v1.Control.CreateActorEgressPolicy:input_type -> substrate.ateapi.v1.CreateActorEgressPolicyRequest - 83, // 204: substrate.ateapi.v1.Control.UpdateActorEgressPolicy:input_type -> substrate.ateapi.v1.UpdateActorEgressPolicyRequest - 84, // 205: substrate.ateapi.v1.Control.DeleteActorEgressPolicy:input_type -> substrate.ateapi.v1.DeleteActorEgressPolicyRequest - 86, // 206: substrate.ateapi.v1.Control.MintActorJwt:input_type -> substrate.ateapi.v1.MintActorJwtRequest - 88, // 207: substrate.ateapi.v1.Control.MintActorCertificate:input_type -> substrate.ateapi.v1.MintActorCertificateRequest - 92, // 208: substrate.ateapi.v1.Control.CreateTag:input_type -> substrate.ateapi.v1.CreateTagRequest - 85, // 209: substrate.ateapi.v1.Control.GetTag:input_type -> substrate.ateapi.v1.GetTagRequest - 90, // 210: substrate.ateapi.v1.Control.ListTags:input_type -> substrate.ateapi.v1.ListTagsRequest - 93, // 211: substrate.ateapi.v1.Control.UpdateTag:input_type -> substrate.ateapi.v1.UpdateTagRequest - 94, // 212: substrate.ateapi.v1.Control.DeleteTag:input_type -> substrate.ateapi.v1.DeleteTagRequest - 99, // 213: substrate.ateapi.v1.Control.ListWorkers:input_type -> substrate.ateapi.v1.ListWorkersRequest - 101, // 214: substrate.ateapi.v1.Control.GetWorker:input_type -> substrate.ateapi.v1.GetWorkerRequest - 102, // 215: substrate.ateapi.v1.Control.CreateWorker:input_type -> substrate.ateapi.v1.CreateWorkerRequest - 103, // 216: substrate.ateapi.v1.Control.UpdateWorker:input_type -> substrate.ateapi.v1.UpdateWorkerRequest - 104, // 217: substrate.ateapi.v1.Control.DeleteWorker:input_type -> substrate.ateapi.v1.DeleteWorkerRequest - 105, // 218: substrate.ateapi.v1.Control.DrainWorker:input_type -> substrate.ateapi.v1.DrainWorkerRequest - 107, // 219: substrate.ateapi.v1.Control.MarkWorkerAvailable:input_type -> substrate.ateapi.v1.MarkWorkerAvailableRequest - 109, // 220: substrate.ateapi.v1.Control.MarkWorkerUnavailable:input_type -> substrate.ateapi.v1.MarkWorkerUnavailableRequest - 97, // 221: substrate.ateapi.v1.Control.ListWorkerActorAssignments:input_type -> substrate.ateapi.v1.ListWorkerActorAssignmentsRequest - 111, // 222: substrate.ateapi.v1.Control.ListActors:input_type -> substrate.ateapi.v1.ListActorsRequest - 60, // 223: substrate.ateapi.v1.Control.CreateAtespace:input_type -> substrate.ateapi.v1.CreateAtespaceRequest - 61, // 224: substrate.ateapi.v1.Control.GetAtespace:input_type -> substrate.ateapi.v1.GetAtespaceRequest - 62, // 225: substrate.ateapi.v1.Control.ListAtespaces:input_type -> substrate.ateapi.v1.ListAtespacesRequest - 64, // 226: substrate.ateapi.v1.Control.DeleteAtespace:input_type -> substrate.ateapi.v1.DeleteAtespaceRequest - 65, // 227: substrate.ateapi.v1.Control.CreateActorTemplate:input_type -> substrate.ateapi.v1.CreateActorTemplateRequest - 66, // 228: substrate.ateapi.v1.Control.GetActorTemplate:input_type -> substrate.ateapi.v1.GetActorTemplateRequest - 67, // 229: substrate.ateapi.v1.Control.ListActorTemplates:input_type -> substrate.ateapi.v1.ListActorTemplatesRequest - 69, // 230: substrate.ateapi.v1.Control.DeleteActorTemplate:input_type -> substrate.ateapi.v1.DeleteActorTemplateRequest - 142, // 231: substrate.ateapi.v1.Control.GetGlobalAccessPolicy:input_type -> substrate.ateapi.v1.GetGlobalAccessPolicyRequest - 143, // 232: substrate.ateapi.v1.Control.CreateGlobalAccessPolicy:input_type -> substrate.ateapi.v1.CreateGlobalAccessPolicyRequest - 144, // 233: substrate.ateapi.v1.Control.UpdateGlobalAccessPolicy:input_type -> substrate.ateapi.v1.UpdateGlobalAccessPolicyRequest - 145, // 234: substrate.ateapi.v1.Control.GetAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.GetAtespaceAccessPolicyRequest - 146, // 235: substrate.ateapi.v1.Control.CreateAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest - 147, // 236: substrate.ateapi.v1.Control.UpdateAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest - 148, // 237: substrate.ateapi.v1.Control.DeleteAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest - 122, // 238: substrate.ateapi.v1.WorkerService.AcquireWorkerEpoch:input_type -> substrate.ateapi.v1.AcquireWorkerEpochRequest - 124, // 239: substrate.ateapi.v1.WorkerService.RegisterWorker:input_type -> substrate.ateapi.v1.RegisterWorkerRequest - 126, // 240: substrate.ateapi.v1.WorkerService.SetWorkerSandboxRuntimes:input_type -> substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest - 128, // 241: substrate.ateapi.v1.WorkerService.MintWorkerActorCertificate:input_type -> substrate.ateapi.v1.MintWorkerActorCertificateRequest - 130, // 242: substrate.ateapi.v1.WorkerService.MintWorkerActorJwt:input_type -> substrate.ateapi.v1.MintWorkerActorJwtRequest - 132, // 243: substrate.ateapi.v1.WorkerService.RequestActorSuspend:input_type -> substrate.ateapi.v1.RequestActorSuspendRequest - 136, // 244: substrate.ateapi.v1.WorkerService.ReportActorSnapshotState:input_type -> substrate.ateapi.v1.ReportActorSnapshotStateRequest - 134, // 245: substrate.ateapi.v1.WorkerService.ReportActorCrashed:input_type -> substrate.ateapi.v1.ReportActorCrashedRequest - 17, // 246: substrate.ateapi.v1.Control.GetActor:output_type -> substrate.ateapi.v1.Actor - 17, // 247: substrate.ateapi.v1.Control.CreateActor:output_type -> substrate.ateapi.v1.Actor - 17, // 248: substrate.ateapi.v1.Control.UpdateActor:output_type -> substrate.ateapi.v1.Actor - 74, // 249: substrate.ateapi.v1.Control.SuspendActor:output_type -> substrate.ateapi.v1.SuspendActorResponse - 77, // 250: substrate.ateapi.v1.Control.ResumeActor:output_type -> substrate.ateapi.v1.ResumeActorResponse - 79, // 251: substrate.ateapi.v1.Control.RevertActor:output_type -> substrate.ateapi.v1.RevertActorResponse - 17, // 252: substrate.ateapi.v1.Control.DeleteActor:output_type -> substrate.ateapi.v1.Actor - 18, // 253: substrate.ateapi.v1.Control.GetActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy - 18, // 254: substrate.ateapi.v1.Control.CreateActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy - 18, // 255: substrate.ateapi.v1.Control.UpdateActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy - 18, // 256: substrate.ateapi.v1.Control.DeleteActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy - 87, // 257: substrate.ateapi.v1.Control.MintActorJwt:output_type -> substrate.ateapi.v1.MintActorJwtResponse - 89, // 258: substrate.ateapi.v1.Control.MintActorCertificate:output_type -> substrate.ateapi.v1.MintActorCertificateResponse - 33, // 259: substrate.ateapi.v1.Control.CreateTag:output_type -> substrate.ateapi.v1.Tag - 33, // 260: substrate.ateapi.v1.Control.GetTag:output_type -> substrate.ateapi.v1.Tag - 91, // 261: substrate.ateapi.v1.Control.ListTags:output_type -> substrate.ateapi.v1.ListTagsResponse - 33, // 262: substrate.ateapi.v1.Control.UpdateTag:output_type -> substrate.ateapi.v1.Tag - 33, // 263: substrate.ateapi.v1.Control.DeleteTag:output_type -> substrate.ateapi.v1.Tag - 100, // 264: substrate.ateapi.v1.Control.ListWorkers:output_type -> substrate.ateapi.v1.ListWorkersResponse - 113, // 265: substrate.ateapi.v1.Control.GetWorker:output_type -> substrate.ateapi.v1.Worker - 113, // 266: substrate.ateapi.v1.Control.CreateWorker:output_type -> substrate.ateapi.v1.Worker - 113, // 267: substrate.ateapi.v1.Control.UpdateWorker:output_type -> substrate.ateapi.v1.Worker - 113, // 268: substrate.ateapi.v1.Control.DeleteWorker:output_type -> substrate.ateapi.v1.Worker - 106, // 269: substrate.ateapi.v1.Control.DrainWorker:output_type -> substrate.ateapi.v1.DrainWorkerResponse - 108, // 270: substrate.ateapi.v1.Control.MarkWorkerAvailable:output_type -> substrate.ateapi.v1.MarkWorkerAvailableResponse - 110, // 271: substrate.ateapi.v1.Control.MarkWorkerUnavailable:output_type -> substrate.ateapi.v1.MarkWorkerUnavailableResponse - 98, // 272: substrate.ateapi.v1.Control.ListWorkerActorAssignments:output_type -> substrate.ateapi.v1.ListWorkerActorAssignmentsResponse - 112, // 273: substrate.ateapi.v1.Control.ListActors:output_type -> substrate.ateapi.v1.ListActorsResponse - 34, // 274: substrate.ateapi.v1.Control.CreateAtespace:output_type -> substrate.ateapi.v1.Atespace - 34, // 275: substrate.ateapi.v1.Control.GetAtespace:output_type -> substrate.ateapi.v1.Atespace - 63, // 276: substrate.ateapi.v1.Control.ListAtespaces:output_type -> substrate.ateapi.v1.ListAtespacesResponse - 34, // 277: substrate.ateapi.v1.Control.DeleteAtespace:output_type -> substrate.ateapi.v1.Atespace - 36, // 278: substrate.ateapi.v1.Control.CreateActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate - 36, // 279: substrate.ateapi.v1.Control.GetActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate - 68, // 280: substrate.ateapi.v1.Control.ListActorTemplates:output_type -> substrate.ateapi.v1.ListActorTemplatesResponse - 36, // 281: substrate.ateapi.v1.Control.DeleteActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate - 140, // 282: substrate.ateapi.v1.Control.GetGlobalAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 140, // 283: substrate.ateapi.v1.Control.CreateGlobalAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 140, // 284: substrate.ateapi.v1.Control.UpdateGlobalAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 140, // 285: substrate.ateapi.v1.Control.GetAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 140, // 286: substrate.ateapi.v1.Control.CreateAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 140, // 287: substrate.ateapi.v1.Control.UpdateAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 140, // 288: substrate.ateapi.v1.Control.DeleteAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 123, // 289: substrate.ateapi.v1.WorkerService.AcquireWorkerEpoch:output_type -> substrate.ateapi.v1.AcquireWorkerEpochResponse - 125, // 290: substrate.ateapi.v1.WorkerService.RegisterWorker:output_type -> substrate.ateapi.v1.RegisterWorkerResponse - 127, // 291: substrate.ateapi.v1.WorkerService.SetWorkerSandboxRuntimes:output_type -> substrate.ateapi.v1.SetWorkerSandboxRuntimesResponse - 129, // 292: substrate.ateapi.v1.WorkerService.MintWorkerActorCertificate:output_type -> substrate.ateapi.v1.MintWorkerActorCertificateResponse - 131, // 293: substrate.ateapi.v1.WorkerService.MintWorkerActorJwt:output_type -> substrate.ateapi.v1.MintWorkerActorJwtResponse - 133, // 294: substrate.ateapi.v1.WorkerService.RequestActorSuspend:output_type -> substrate.ateapi.v1.RequestActorSuspendResponse - 137, // 295: substrate.ateapi.v1.WorkerService.ReportActorSnapshotState:output_type -> substrate.ateapi.v1.ReportActorSnapshotStateResponse - 135, // 296: substrate.ateapi.v1.WorkerService.ReportActorCrashed:output_type -> substrate.ateapi.v1.ReportActorCrashedResponse - 246, // [246:297] is the sub-list for method output_type - 195, // [195:246] is the sub-list for method input_type - 193, // [193:195] is the sub-list for extension type_name - 191, // [191:193] is the sub-list for extension extendee - 0, // [0:191] is the sub-list for field type_name + 119, // 146: substrate.ateapi.v1.WorkerSandboxRuntime.version:type_name -> substrate.ateapi.v1.VersionedSandboxCompat + 117, // 147: substrate.ateapi.v1.WorkerSandboxRuntime.actor_overhead:type_name -> substrate.ateapi.v1.RuntimeOverhead + 119, // 148: substrate.ateapi.v1.ActorSandboxRuntime.version:type_name -> substrate.ateapi.v1.VersionedSandboxCompat + 120, // 149: substrate.ateapi.v1.VersionedSandboxCompat.attributes:type_name -> substrate.ateapi.v1.AttributeEntry + 14, // 150: substrate.ateapi.v1.ActorAssignment.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 35, // 151: substrate.ateapi.v1.ActorAssignment.actor:type_name -> substrate.ateapi.v1.ObjectRef + 38, // 152: substrate.ateapi.v1.ActorAssignment.resources:type_name -> substrate.ateapi.v1.Resources + 35, // 153: substrate.ateapi.v1.AcquireWorkerEpochRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef + 35, // 154: substrate.ateapi.v1.RegisterWorkerRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef + 121, // 155: substrate.ateapi.v1.RegisterWorkerRequest.capacity:type_name -> substrate.ateapi.v1.WorkerResources + 113, // 156: substrate.ateapi.v1.RegisterWorkerResponse.worker:type_name -> substrate.ateapi.v1.Worker + 35, // 157: substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef + 115, // 158: substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest.sandbox_classes:type_name -> substrate.ateapi.v1.WorkerSandboxClass + 113, // 159: substrate.ateapi.v1.SetWorkerSandboxRuntimesResponse.worker:type_name -> substrate.ateapi.v1.Worker + 35, // 160: substrate.ateapi.v1.MintWorkerActorCertificateRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 35, // 161: substrate.ateapi.v1.MintWorkerActorCertificateRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef + 35, // 162: substrate.ateapi.v1.MintWorkerActorJwtRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 155, // 163: substrate.ateapi.v1.MintWorkerActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp + 35, // 164: substrate.ateapi.v1.RequestActorSuspendRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef + 35, // 165: substrate.ateapi.v1.RequestActorSuspendRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 17, // 166: substrate.ateapi.v1.RequestActorSuspendResponse.actor:type_name -> substrate.ateapi.v1.Actor + 35, // 167: substrate.ateapi.v1.ReportActorCrashedRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef + 35, // 168: substrate.ateapi.v1.ReportActorCrashedRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 17, // 169: substrate.ateapi.v1.ReportActorCrashedResponse.actor:type_name -> substrate.ateapi.v1.Actor + 35, // 170: substrate.ateapi.v1.ReportActorSnapshotStateRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 139, // 171: substrate.ateapi.v1.ReportActorSnapshotStateRequest.state:type_name -> substrate.ateapi.v1.ActorSnapshotState + 140, // 172: substrate.ateapi.v1.ActorSnapshotState.latest_local:type_name -> substrate.ateapi.v1.SnapshotReport + 140, // 173: substrate.ateapi.v1.ActorSnapshotState.uploading:type_name -> substrate.ateapi.v1.SnapshotReport + 140, // 174: substrate.ateapi.v1.ActorSnapshotState.latest_durable:type_name -> substrate.ateapi.v1.SnapshotReport + 155, // 175: substrate.ateapi.v1.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp + 2, // 176: substrate.ateapi.v1.SnapshotReport.fidelity:type_name -> substrate.ateapi.v1.SnapshotFidelity + 10, // 177: substrate.ateapi.v1.SnapshotReport.object:type_name -> substrate.ateapi.v1.ObjectSnapshot + 155, // 178: substrate.ateapi.v1.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp + 155, // 179: substrate.ateapi.v1.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp + 14, // 180: substrate.ateapi.v1.AccessPolicy.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 142, // 181: substrate.ateapi.v1.AccessPolicy.bindings:type_name -> substrate.ateapi.v1.Binding + 141, // 182: substrate.ateapi.v1.CreateGlobalAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy + 141, // 183: substrate.ateapi.v1.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy + 35, // 184: substrate.ateapi.v1.GetAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef + 35, // 185: substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef + 141, // 186: substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy + 35, // 187: substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef + 141, // 188: substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy + 35, // 189: substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef + 95, // 190: substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions + 9, // 191: substrate.ateapi.v1.ResourceAnnotation.scope:type_name -> substrate.ateapi.v1.Scope + 156, // 192: substrate.ateapi.v1.resource:extendee -> google.protobuf.MessageOptions + 157, // 193: substrate.ateapi.v1.method:extendee -> google.protobuf.MethodOptions + 150, // 194: substrate.ateapi.v1.resource:type_name -> substrate.ateapi.v1.ResourceAnnotation + 151, // 195: substrate.ateapi.v1.method:type_name -> substrate.ateapi.v1.MethodAnnotation + 70, // 196: substrate.ateapi.v1.Control.GetActor:input_type -> substrate.ateapi.v1.GetActorRequest + 71, // 197: substrate.ateapi.v1.Control.CreateActor:input_type -> substrate.ateapi.v1.CreateActorRequest + 72, // 198: substrate.ateapi.v1.Control.UpdateActor:input_type -> substrate.ateapi.v1.UpdateActorRequest + 73, // 199: substrate.ateapi.v1.Control.SuspendActor:input_type -> substrate.ateapi.v1.SuspendActorRequest + 76, // 200: substrate.ateapi.v1.Control.ResumeActor:input_type -> substrate.ateapi.v1.ResumeActorRequest + 78, // 201: substrate.ateapi.v1.Control.RevertActor:input_type -> substrate.ateapi.v1.RevertActorRequest + 80, // 202: substrate.ateapi.v1.Control.DeleteActor:input_type -> substrate.ateapi.v1.DeleteActorRequest + 81, // 203: substrate.ateapi.v1.Control.GetActorEgressPolicy:input_type -> substrate.ateapi.v1.GetActorEgressPolicyRequest + 82, // 204: substrate.ateapi.v1.Control.CreateActorEgressPolicy:input_type -> substrate.ateapi.v1.CreateActorEgressPolicyRequest + 83, // 205: substrate.ateapi.v1.Control.UpdateActorEgressPolicy:input_type -> substrate.ateapi.v1.UpdateActorEgressPolicyRequest + 84, // 206: substrate.ateapi.v1.Control.DeleteActorEgressPolicy:input_type -> substrate.ateapi.v1.DeleteActorEgressPolicyRequest + 86, // 207: substrate.ateapi.v1.Control.MintActorJwt:input_type -> substrate.ateapi.v1.MintActorJwtRequest + 88, // 208: substrate.ateapi.v1.Control.MintActorCertificate:input_type -> substrate.ateapi.v1.MintActorCertificateRequest + 92, // 209: substrate.ateapi.v1.Control.CreateTag:input_type -> substrate.ateapi.v1.CreateTagRequest + 85, // 210: substrate.ateapi.v1.Control.GetTag:input_type -> substrate.ateapi.v1.GetTagRequest + 90, // 211: substrate.ateapi.v1.Control.ListTags:input_type -> substrate.ateapi.v1.ListTagsRequest + 93, // 212: substrate.ateapi.v1.Control.UpdateTag:input_type -> substrate.ateapi.v1.UpdateTagRequest + 94, // 213: substrate.ateapi.v1.Control.DeleteTag:input_type -> substrate.ateapi.v1.DeleteTagRequest + 99, // 214: substrate.ateapi.v1.Control.ListWorkers:input_type -> substrate.ateapi.v1.ListWorkersRequest + 101, // 215: substrate.ateapi.v1.Control.GetWorker:input_type -> substrate.ateapi.v1.GetWorkerRequest + 102, // 216: substrate.ateapi.v1.Control.CreateWorker:input_type -> substrate.ateapi.v1.CreateWorkerRequest + 103, // 217: substrate.ateapi.v1.Control.UpdateWorker:input_type -> substrate.ateapi.v1.UpdateWorkerRequest + 104, // 218: substrate.ateapi.v1.Control.DeleteWorker:input_type -> substrate.ateapi.v1.DeleteWorkerRequest + 105, // 219: substrate.ateapi.v1.Control.DrainWorker:input_type -> substrate.ateapi.v1.DrainWorkerRequest + 107, // 220: substrate.ateapi.v1.Control.MarkWorkerAvailable:input_type -> substrate.ateapi.v1.MarkWorkerAvailableRequest + 109, // 221: substrate.ateapi.v1.Control.MarkWorkerUnavailable:input_type -> substrate.ateapi.v1.MarkWorkerUnavailableRequest + 97, // 222: substrate.ateapi.v1.Control.ListWorkerActorAssignments:input_type -> substrate.ateapi.v1.ListWorkerActorAssignmentsRequest + 111, // 223: substrate.ateapi.v1.Control.ListActors:input_type -> substrate.ateapi.v1.ListActorsRequest + 60, // 224: substrate.ateapi.v1.Control.CreateAtespace:input_type -> substrate.ateapi.v1.CreateAtespaceRequest + 61, // 225: substrate.ateapi.v1.Control.GetAtespace:input_type -> substrate.ateapi.v1.GetAtespaceRequest + 62, // 226: substrate.ateapi.v1.Control.ListAtespaces:input_type -> substrate.ateapi.v1.ListAtespacesRequest + 64, // 227: substrate.ateapi.v1.Control.DeleteAtespace:input_type -> substrate.ateapi.v1.DeleteAtespaceRequest + 65, // 228: substrate.ateapi.v1.Control.CreateActorTemplate:input_type -> substrate.ateapi.v1.CreateActorTemplateRequest + 66, // 229: substrate.ateapi.v1.Control.GetActorTemplate:input_type -> substrate.ateapi.v1.GetActorTemplateRequest + 67, // 230: substrate.ateapi.v1.Control.ListActorTemplates:input_type -> substrate.ateapi.v1.ListActorTemplatesRequest + 69, // 231: substrate.ateapi.v1.Control.DeleteActorTemplate:input_type -> substrate.ateapi.v1.DeleteActorTemplateRequest + 143, // 232: substrate.ateapi.v1.Control.GetGlobalAccessPolicy:input_type -> substrate.ateapi.v1.GetGlobalAccessPolicyRequest + 144, // 233: substrate.ateapi.v1.Control.CreateGlobalAccessPolicy:input_type -> substrate.ateapi.v1.CreateGlobalAccessPolicyRequest + 145, // 234: substrate.ateapi.v1.Control.UpdateGlobalAccessPolicy:input_type -> substrate.ateapi.v1.UpdateGlobalAccessPolicyRequest + 146, // 235: substrate.ateapi.v1.Control.GetAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.GetAtespaceAccessPolicyRequest + 147, // 236: substrate.ateapi.v1.Control.CreateAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest + 148, // 237: substrate.ateapi.v1.Control.UpdateAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest + 149, // 238: substrate.ateapi.v1.Control.DeleteAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest + 123, // 239: substrate.ateapi.v1.WorkerService.AcquireWorkerEpoch:input_type -> substrate.ateapi.v1.AcquireWorkerEpochRequest + 125, // 240: substrate.ateapi.v1.WorkerService.RegisterWorker:input_type -> substrate.ateapi.v1.RegisterWorkerRequest + 127, // 241: substrate.ateapi.v1.WorkerService.SetWorkerSandboxRuntimes:input_type -> substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest + 129, // 242: substrate.ateapi.v1.WorkerService.MintWorkerActorCertificate:input_type -> substrate.ateapi.v1.MintWorkerActorCertificateRequest + 131, // 243: substrate.ateapi.v1.WorkerService.MintWorkerActorJwt:input_type -> substrate.ateapi.v1.MintWorkerActorJwtRequest + 133, // 244: substrate.ateapi.v1.WorkerService.RequestActorSuspend:input_type -> substrate.ateapi.v1.RequestActorSuspendRequest + 137, // 245: substrate.ateapi.v1.WorkerService.ReportActorSnapshotState:input_type -> substrate.ateapi.v1.ReportActorSnapshotStateRequest + 135, // 246: substrate.ateapi.v1.WorkerService.ReportActorCrashed:input_type -> substrate.ateapi.v1.ReportActorCrashedRequest + 17, // 247: substrate.ateapi.v1.Control.GetActor:output_type -> substrate.ateapi.v1.Actor + 17, // 248: substrate.ateapi.v1.Control.CreateActor:output_type -> substrate.ateapi.v1.Actor + 17, // 249: substrate.ateapi.v1.Control.UpdateActor:output_type -> substrate.ateapi.v1.Actor + 74, // 250: substrate.ateapi.v1.Control.SuspendActor:output_type -> substrate.ateapi.v1.SuspendActorResponse + 77, // 251: substrate.ateapi.v1.Control.ResumeActor:output_type -> substrate.ateapi.v1.ResumeActorResponse + 79, // 252: substrate.ateapi.v1.Control.RevertActor:output_type -> substrate.ateapi.v1.RevertActorResponse + 17, // 253: substrate.ateapi.v1.Control.DeleteActor:output_type -> substrate.ateapi.v1.Actor + 18, // 254: substrate.ateapi.v1.Control.GetActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy + 18, // 255: substrate.ateapi.v1.Control.CreateActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy + 18, // 256: substrate.ateapi.v1.Control.UpdateActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy + 18, // 257: substrate.ateapi.v1.Control.DeleteActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy + 87, // 258: substrate.ateapi.v1.Control.MintActorJwt:output_type -> substrate.ateapi.v1.MintActorJwtResponse + 89, // 259: substrate.ateapi.v1.Control.MintActorCertificate:output_type -> substrate.ateapi.v1.MintActorCertificateResponse + 33, // 260: substrate.ateapi.v1.Control.CreateTag:output_type -> substrate.ateapi.v1.Tag + 33, // 261: substrate.ateapi.v1.Control.GetTag:output_type -> substrate.ateapi.v1.Tag + 91, // 262: substrate.ateapi.v1.Control.ListTags:output_type -> substrate.ateapi.v1.ListTagsResponse + 33, // 263: substrate.ateapi.v1.Control.UpdateTag:output_type -> substrate.ateapi.v1.Tag + 33, // 264: substrate.ateapi.v1.Control.DeleteTag:output_type -> substrate.ateapi.v1.Tag + 100, // 265: substrate.ateapi.v1.Control.ListWorkers:output_type -> substrate.ateapi.v1.ListWorkersResponse + 113, // 266: substrate.ateapi.v1.Control.GetWorker:output_type -> substrate.ateapi.v1.Worker + 113, // 267: substrate.ateapi.v1.Control.CreateWorker:output_type -> substrate.ateapi.v1.Worker + 113, // 268: substrate.ateapi.v1.Control.UpdateWorker:output_type -> substrate.ateapi.v1.Worker + 113, // 269: substrate.ateapi.v1.Control.DeleteWorker:output_type -> substrate.ateapi.v1.Worker + 106, // 270: substrate.ateapi.v1.Control.DrainWorker:output_type -> substrate.ateapi.v1.DrainWorkerResponse + 108, // 271: substrate.ateapi.v1.Control.MarkWorkerAvailable:output_type -> substrate.ateapi.v1.MarkWorkerAvailableResponse + 110, // 272: substrate.ateapi.v1.Control.MarkWorkerUnavailable:output_type -> substrate.ateapi.v1.MarkWorkerUnavailableResponse + 98, // 273: substrate.ateapi.v1.Control.ListWorkerActorAssignments:output_type -> substrate.ateapi.v1.ListWorkerActorAssignmentsResponse + 112, // 274: substrate.ateapi.v1.Control.ListActors:output_type -> substrate.ateapi.v1.ListActorsResponse + 34, // 275: substrate.ateapi.v1.Control.CreateAtespace:output_type -> substrate.ateapi.v1.Atespace + 34, // 276: substrate.ateapi.v1.Control.GetAtespace:output_type -> substrate.ateapi.v1.Atespace + 63, // 277: substrate.ateapi.v1.Control.ListAtespaces:output_type -> substrate.ateapi.v1.ListAtespacesResponse + 34, // 278: substrate.ateapi.v1.Control.DeleteAtespace:output_type -> substrate.ateapi.v1.Atespace + 36, // 279: substrate.ateapi.v1.Control.CreateActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 36, // 280: substrate.ateapi.v1.Control.GetActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 68, // 281: substrate.ateapi.v1.Control.ListActorTemplates:output_type -> substrate.ateapi.v1.ListActorTemplatesResponse + 36, // 282: substrate.ateapi.v1.Control.DeleteActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 141, // 283: substrate.ateapi.v1.Control.GetGlobalAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 141, // 284: substrate.ateapi.v1.Control.CreateGlobalAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 141, // 285: substrate.ateapi.v1.Control.UpdateGlobalAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 141, // 286: substrate.ateapi.v1.Control.GetAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 141, // 287: substrate.ateapi.v1.Control.CreateAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 141, // 288: substrate.ateapi.v1.Control.UpdateAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 141, // 289: substrate.ateapi.v1.Control.DeleteAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 124, // 290: substrate.ateapi.v1.WorkerService.AcquireWorkerEpoch:output_type -> substrate.ateapi.v1.AcquireWorkerEpochResponse + 126, // 291: substrate.ateapi.v1.WorkerService.RegisterWorker:output_type -> substrate.ateapi.v1.RegisterWorkerResponse + 128, // 292: substrate.ateapi.v1.WorkerService.SetWorkerSandboxRuntimes:output_type -> substrate.ateapi.v1.SetWorkerSandboxRuntimesResponse + 130, // 293: substrate.ateapi.v1.WorkerService.MintWorkerActorCertificate:output_type -> substrate.ateapi.v1.MintWorkerActorCertificateResponse + 132, // 294: substrate.ateapi.v1.WorkerService.MintWorkerActorJwt:output_type -> substrate.ateapi.v1.MintWorkerActorJwtResponse + 134, // 295: substrate.ateapi.v1.WorkerService.RequestActorSuspend:output_type -> substrate.ateapi.v1.RequestActorSuspendResponse + 138, // 296: substrate.ateapi.v1.WorkerService.ReportActorSnapshotState:output_type -> substrate.ateapi.v1.ReportActorSnapshotStateResponse + 136, // 297: substrate.ateapi.v1.WorkerService.ReportActorCrashed:output_type -> substrate.ateapi.v1.ReportActorCrashedResponse + 247, // [247:298] is the sub-list for method output_type + 196, // [196:247] is the sub-list for method input_type + 194, // [194:196] is the sub-list for extension type_name + 192, // [192:194] is the sub-list for extension extendee + 0, // [0:192] is the sub-list for field type_name } func init() { file_api_ateapi_v1_ateapi_proto_init() } @@ -11172,7 +11262,7 @@ func file_api_ateapi_v1_ateapi_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_api_ateapi_v1_ateapi_proto_rawDesc), len(file_api_ateapi_v1_ateapi_proto_rawDesc)), NumEnums: 10, - NumMessages: 144, + NumMessages: 145, NumExtensions: 2, NumServices: 2, }, diff --git a/api/ateapi/v1/ateapi.proto b/api/ateapi/v1/ateapi.proto index a5e777a4a6..7a59f6a098 100644 --- a/api/ateapi/v1/ateapi.proto +++ b/api/ateapi/v1/ateapi.proto @@ -291,24 +291,24 @@ enum SnapshotOwner { // Keep this in sync with Snapshot.owner's maximum. } -// SnapshotStorageStatus indicates the lifecycle stage of a durable snapshot. -enum SnapshotStorageStatus { - SNAPSHOT_STORAGE_STATUS_UNSPECIFIED = 0; +// SnapshotStorageState indicates the lifecycle stage of a durable snapshot. +enum SnapshotStorageState { + SNAPSHOT_STORAGE_STATE_UNSPECIFIED = 0; // Snapshot creation in durable storage is in progress. - SNAPSHOT_STORAGE_STATUS_PENDING = 1; + SNAPSHOT_STORAGE_STATE_PENDING = 1; // Snapshot is complete and ready for use in durable storage. - SNAPSHOT_STORAGE_STATUS_COMPLETED = 2; - // Keep this in sync with SnapshotStorage.status's maximum. + SNAPSHOT_STORAGE_STATE_COMPLETED = 2; + // Keep this in sync with SnapshotStorage.state's maximum. } // SnapshotStorage records the durable storage representation of a Snapshot. message SnapshotStorage { - // status is the lifecycle status of the durable snapshot. + // state is the lifecycle state of the durable snapshot. // // +k8s:required // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotStorageStatus enum - SnapshotStorageStatus status = 1; + // +k8s:maximum=2 # keep this in sync with the SnapshotStorageState enum + SnapshotStorageState state = 1; // object snapshot is stored in object storage. // @@ -1279,9 +1279,10 @@ message SnapshotConfig { // storage_location is the base object-storage URI snapshots of actors on // this version are stored under. If unset, ate-api-server's // --snapshot-location is used. Requires the PerTemplateSnapshotLocation preview - // gate; ignored without it. + // gate. // // +ate:preview=PerTemplateSnapshotLocation + // +k8s:ifDisabled(PerTemplateSnapshotLocation)=+k8s:forbidden // +k8s:optional // +k8s:maxLength=1024 // +k8s:customValidation # Validate URI @@ -2409,7 +2410,8 @@ message WorkerStatus { WorkerResources capacity = 2; // The allocated resources of the Worker. This is the sum of all the - // resources consumed by the Actors currently assigned to this Worker. + // resources consumed by the Actors currently assigned to this Worker, + // including their runtime's actor_overhead. // // +k8s:optional WorkerResources allocated = 3; @@ -2505,10 +2507,39 @@ message WorkerSandboxRuntime { // +k8s:required VersionedSandboxCompat version = 1; + // The resource overhead incurred by a Worker for each Actor running under + // this runtime, on top of the Actor's stated limits. Placement considers + // this when deciding if a Worker has enough capacity for an Actor, and it is + // accounted for in allocated. If not specified, Actors under this runtime + // are assumed to have negligible overhead. + // + // +k8s:optional + RuntimeOverhead actor_overhead = 2; + // TODO: Add an informational SandboxConfig version name (e.g. "gvisor-2") // once workers report one. } +// RuntimeOverhead is the memory an Actor uses on its Worker outside its own +// limits: a fixed amount, plus an amount in proportion to its memory limit. +message RuntimeOverhead { + // memory_bytes is a fixed amount of memory for each Actor. + // + // +k8s:optional + // +k8s:minimum=0 + // +k8s:maximum=1099511627776 # 1TiB + int64 memory_bytes = 1; + + // memory_bytes_per_gib is memory per GiB of the Actor's memory limit, e.g. + // host page tables for a microVM's guest RAM. It scales in proportion and + // rounds down to a byte, so an Actor with a 512MiB limit adds half of it. + // + // +k8s:optional + // +k8s:minimum=0 + // +k8s:maximum=1073741824 # 1GiB + int64 memory_bytes_per_gib = 2; +} + // ActorSandboxRuntime is the runtime an actor runs under, as recorded on its // assignment and on the snapshots taken from it: one class and one of the // compatibility versions a Worker reports for it. @@ -2630,7 +2661,8 @@ message ActorAssignment { // +k8s:format=k8s-uuid string actor_uid = 3; - // The resources allocated to the Actor when it was assigned. + // The resources allocated to the Actor when it was assigned: its limits, + // plus the actor_overhead of the runtime it runs under on the Worker. // // +k8s:optional Resources resources = 4; diff --git a/benchmarking/README.md b/benchmarking/README.md index c584d1aad3..50a9d9d397 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -1,945 +1 @@ -# Substrate Benchmarking - -This is the nascent suite for benchmarking Substrate's performance at scale. - -The suite also measures the telemetry volume and the capacity of the OTel -collector: how much trace data and metric data substrate and its actors send, -and if the collector can accept it. To make a measurement, read -[telemetry/README.md](telemetry/README.md). For the prerequisites and the -scenario ladder, read [observability.md](observability.md). - -## Deploy benchmarks - -> [!IMPORTANT] -> Source the environment configuration file (e.g., `source .ate-dev-env.sh`) -> first so `PROJECT_ID`, `BUCKET_NAME`, etc. are set. -> -> **For Local Kind Clusters**: Set `KO_DOCKER_REPO` to the local registry so images are pulled without remote cloud credentials: -> ```bash -> export KO_DOCKER_REPO="localhost:5001" -> export BUCKET_NAME="ate-snapshots" -> ``` - -> [!IMPORTANT] -> The ate-api-server checks every call against -> [authorization](../docs/authorization.md), and Locust calls it as the -> `benchmarking/default` ServiceAccount. The Envoy ingress router also requires -> an allowlist of client SPIFFE IDs at install time. Install Substrate with -> `benchmarking/default` in both settings alongside the default `ate-client` -> owner: -> ```bash -> ATE_API_AUTHZ_BOOTSTRAP_OWNERS=system:serviceaccount:ate-system:ate-client,spiffe://cluster.local/ns/benchmarking/sa/default \ -> ATE_INGRESS_ALLOWED_SPIFFE_IDS=spiffe://cluster.local/ns/benchmarking/sa/default \ -> ./hack/install-ate.sh deploy ate-system -> ``` -> The nightly automation sets both itself; see -> [automation/README.md](automation/README.md). - -Note that deploying the benchmarks does not run them. You must visit Locust's -web UI to start a test. - -A single wrapper deploys the scale workloads, builds and pushes the Locust -image, then deploys the Locust workers: - -```bash -./benchmarking/deploy_locust.sh --deploy -``` - -Useful flags: - -* `--worker-count N` — number of `WorkerPool` replicas (default 1). -* `--worker-memory SIZE` — memory request and limit for each `WorkerPool` pod - (default unset, the pod is unsized). Set it above half a node's allocatable - memory to get one worker per node. -* `--skip-build` — reuse the existing `:latest` locust image (skip the - `docker build && docker push` step). - -To tear everything down (locust then workloads, in reverse order): - -```bash -./benchmarking/deploy_locust.sh --delete -``` - -The same operations are also reachable from the top-level installer for -convenience: - -```bash -go run ./cmd/ate-setup deploy benchmarks -go run ./cmd/ate-setup delete benchmarks -``` - -`--skip-build` and `--worker-memory` are only available when invoking -`benchmarking/deploy_locust.sh` directly. - -### Settings - -`ate-setup` reads these settings in addition to those in -[docs/operator-install.md](../docs/operator-install.md#settings), through the same -flag, environment variable, and configuration document channels. - -| Key | Environment variable | Flag | Default | Applies to | -|--------------------------|-------------------------------|----------------------------|----------|------------------------------------------| -| `benchmark.actorMemory` | `BENCHMARK_ACTOR_MEMORY` | `--benchmark-actor-memory` | — | (global) | -| `benchmark.sandboxClass` | `ATE_BENCHMARK_SANDBOX_CLASS` | `--sandbox-class` | `gvisor` | `deploy benchmarks`, `delete benchmarks` | -| `benchmark.workerCount` | `ATE_BENCHMARK_WORKER_COUNT` | `--worker-count` | `1` | `deploy benchmarks`, `delete benchmarks` | - -## Running Tests - -### Locust Web UI -* Run `kubectl port-forward svc/locust -n benchmarking 8089:8089` -* Visit `http://localhost:8089` in your browser to configure and start the load test. - -The different user classes you can select are different types of load behaviors -you can throw at the system. Note that the "CounterUser" load type requires -that the counter demo be installed. - -You can also configure things like the number of users, how quickly those users -are spawned, the frequency with which requests are made and whether or not tracing is -enabled. - -User classes implemented in boomer rather than Python are selected at deploy -time — the stack runs one per deployment: - -```bash -./benchmarking/locust/deploy.sh --deploy --user-class durdir -``` - -### Headless (automation only) - -`runner.py` runs a test without the web UI, writing CSVs, logs and traces to -`--dest`. The nightly automation submits it as a Job on the test cluster; it is -not a local entry point. See [automation/README.md](automation/README.md). - -```bash -python3 runner.py -f tests/.py -t 1m -u 1 --name --dest /tmp/bench -``` - -Four flags control the optional post-run measurements described in -[Benchmark output files](#benchmark-output-files): - -* `--cluster-facts` / `--no-cluster-facts`: read node capacity and worker pod - count from the Kubernetes API once the run ends, to derive density frontiers, - and worker pod restarts and evictions around the run, for `oom_events`. - On by default. Pass `--no-cluster-facts` to skip Kubernetes API discovery. -* `--prometheus-url`: the Prometheus to harvest server-side telemetry from. - Defaults to the in-cluster service installed by - [Optional: Prometheus + Grafana](#optional-prometheus--grafana). -* `--atelet-lag-s`: how long to wait after the run before reading the - atelet's metrics. Defaults to 70. -* `--phase-breakdown` / `--no-phase-breakdown`: read the atelet and worker pod - logs once the run ends and add the suspend/resume phase percentiles to - `stats.jsonl`. On by default; needs `pods/log` access in `ate-system` and - `benchmark-workloads` (granted in `automation/manifests/runner-job.yaml.tmpl`). - See [analysis/README.md](analysis/README.md#automated-runs). - -Test-specific flags are appended to the same command; see the sections below. - -### DurDir Benchmark - -The DurDir benchmark evaluates actor suspend/resume performance, disk persistence overhead, -and state restoration latency when a durable directory is attached to the actor. - -#### DurDir Configuration Knobs - -* `--durdir-file-size-bytes`: Size in bytes of the data file (default `8388608` = 8 MiB). -* `--resume-mode`: Resume trigger mode: - * `explicit` (default): Client invokes the `ResumeActor` RPC before sending traffic. - * `implicit`: Client sends traffic through the router without an explicit wake RPC, testing traffic-triggered resume. -* `--durdir-read-mode`: Verification read mode: - * `data` (default): Server returns full payload bytes for client-side SHA-256 verification. - * `digest`: Server hashes the file and returns size and digest, reducing network transfer. -* `--durdir-template`: ActorTemplate name: - * `glutton-durdir-data` (default): Attaches a durable data directory without memory snapshot restore. - * `glutton-durdir-full`: Attaches a durable data directory and performs a full memory snapshot restore. - * `glutton-storage`: Attaches an external CSI volume instead of a durable - directory. `deploy_locust.sh` and `workloads/deploy.sh` deploy it only - when `--storage-class-name` names an existing StorageClass. To run it on - NFS, install with `hack/install-ate.sh setup csi nfs` and pass - `--storage-class-name csi-nfs-sc`, the class the e2e tests also use. - -#### DurDir Reported Metrics - -* `DurDirWrite`: Initial truncate-write creating the data file. -* `DurDirServeInitial`: First read immediately following file creation. -* `SuspendActor`: Actor suspend latency (snapshot creation + persistence upload). -* `ResumeActor`: Actor resume latency. -* `DurDirServeAfterResume`: First read after resume (measures page faults / lazy load overhead on restored volume). -* `DurDirServeWarm`: Subsequent read within the same active cycle (cached state baseline). -* `DurDirOverwrite`: In-place file overwrite with checksum verification. - -### Sweperf Benchmark - -The sweperf benchmark replays a recorded SWE-Perf task inside an actor, suspending and -resuming between cycles to measure the cost of actor state transitions under a realistic -agent workload. One task is four cycles by default. Each task runs on a fresh actor: once -the last cycle finishes, the actor is deleted and the user creates a new one for the next -task. - -#### Sweperf Tasks - -Each task is one entry (name, digest-pinned image, step count) in -`workloads/manifests/sweperf/sweperf-images.json`. Its ActorTemplate, `sweperf-`, -is generated into `workloads/manifests/sweperf/generated/` and checked in. To add or change -a task, edit the catalog and regenerate: - -```bash -SWEPERF=benchmarking/workloads/manifests/sweperf -python3 -m venv "${SWEPERF}/venv" && "${SWEPERF}/venv/bin/pip" install -r "${SWEPERF}/requirements.txt" -"${SWEPERF}/venv/bin/python" "${SWEPERF}/generate_sweperf_templates.py" '*' -"${SWEPERF}/venv/bin/python" "${SWEPERF}/generate_sweperf_templates.py" --check # fail if generated/ is missing or stale -hack/verify/benchmarking-python.sh # unit tests, as CI runs them -``` - -Deploy a task and the sweperf user class: - -```bash -WORKLOAD_TEMPLATES=sweperf-django-11099 ./benchmarking/workloads/deploy.sh --deploy -./benchmarking/locust/deploy.sh --deploy --user-class sweperf -``` - -Then start the run with `--sweperf-template sweperf-django-11099` and -`--sweperf-total-steps 15`, set in the Locust web UI or through `LOCUST_SWEPERF_TEMPLATE` -and `LOCUST_SWEPERF_TOTAL_STEPS`. The client takes the step count from that option, not -from the template, so it must match the catalog. - -#### Sweperf Reported Metrics - -All rows are in milliseconds. CEL (command execution latency) is the time the trace commands -ran inside the sandbox, as reported by `replay.py`. - -* `ResumeToFirstExec`: Resume RPC start until the sandbox accepts the cycle's `/execute`. -* `CycleCEL`: CEL for one cycle. -* `TaskCEL`: CEL summed over one task. -* `TaskWallClock`: Client wall clock for one task, excluding inter-cycle think time and actor - startup. -* `CreateAtespace`, `CreateActor`, `ResumeActor`, `SuspendActor`, `DeleteActor`: Server-side - elapsed time for each control-plane RPC from the response trailer, or client time without one. -* `_rtt`: Client round trip for the RPC of the same name, recorded only when the trailer is - present, so network and queueing overhead stays visible separately. -* `Workload_Cycle_`: Client time for cycle `n`: `POST /execute` plus `/status` polling until - the job finishes. Polled every `--sweperf-poll-interval-ms` (default 100), so this row sits - up to one interval above the job's actual end. - -The liveness check on a new actor already has it running, so the first cycle of every task -runs on a freshly started actor and its resume is a no-op. Its successful `ResumeActor`, -`ResumeActor_rtt` and `ResumeToFirstExec` samples are not recorded; failures still are. - -The last cycle of a task is not suspended: the actor is deleted straight from running. A -task of `n` cycles therefore records `n - 1` `SuspendActor` samples, and `TaskWallClock` -does not include a final suspend. `DeleteActor` therefore times deleting a running actor, -including its sandbox teardown, so it is not comparable with older runs that suspended the -actor before deleting it. - -`locust_users` counts live actors rather than virtual users: it rises when a user starts an -actor and falls when that actor is deleted, so it dips between tasks while a user waits to -start its next actor. - -`CreateAtespace` is sent once per worker, not per task, and again only if `CreateActor` -reports the atespace missing. - -When the run ends, shutdown cancels cycles still in flight. Calls cut short this way record -nothing, and the interrupted task records no `TaskCEL` or `TaskWallClock` row, so the end of -a run does not show up as a burst of failures. - -#### Sweperf Log Lines - -The boomer log also carries per-task facts for post-processing such as the goodput script in -`tools/goodput`, which decides from them whether a task was good. The benchmark judges -nothing. Durations are in milliseconds. - -* `Completed sweperf cycle`: `actor`, `cycle`, `cel_ms` (0 when the execute failed), - `wall_ms` (resume start to cycle end), `ok` (the execute succeeded, and so did the suspend - if there was one), `resume_ms` (as on the `ResumeActor` row; omitted for a no-op resume) - and `ttfe_ms` (resume start to the asynchronous ack; omitted without one, and for a no-op - resume even when its execute got one). -* `ResumeActor failed in step`: also carries `wall_ms`. The cycle is retried on the next - iteration, and the failed attempt's time counts toward the task's `wall_ms`. -* `sweperf task record`, one per finished task: `v` (1), `user` (boomer goroutine ID, unique - per boomer process), `actor`, `template`, `cycles` and `steps` (the cycle and trace step - counts the task ran, resolved when it started, so a mid-run change to - `--sweperf-num-cycles` or `--sweperf-total-steps` applies from the next task; `cycles` is - capped at `steps`), `task_start` (when the session started) and `end`, `startup_ms` - (`CreateActor` until the sandbox answers `/status`), `cel_ms` and `wall_ms` (as in `TaskCEL` - and `TaskWallClock`), `think_ms` (think time between the task's cycles), `resume_max_ms` - (slowest non-no-op resume), `ttfe_max_ms` (slowest `ttfe_ms`; omitted when no cycle after - the no-op resume got an asynchronous ack, so always for a one-cycle task), `retries` - (failed resumes), `failed`, and the first failure's `fail_step` (`resume`, `execute` or - `suspend`) and `fail_cycle`. -* `sweperf on_start failed`: also carries `user`, to tie a failed start to its user. -* `sweperf shutdown started`: the worker began tearing down. Startups and cycles in flight - are canceled: interrupted tasks log no task record, and startup failures logged after it - belong to the teardown, not the run. - -### ate-env Benchmark - -The `ate-env` benchmark (`--user-class ate_env`) exercises the -[`ate-env-guest`](https://github.com/agent-substrate/env/tree/main/cmd/ate-env-guest) -sandbox image from `agent-substrate/env`. `workloads/deploy.sh` deploys and -deletes the `ate-env` `ActorTemplate` by default using the pinned release guest -image (see -[`workloads/manifests/ate-env-template.yaml.tmpl`](workloads/manifests/ate-env-template.yaml.tmpl) -to override the guest image with a custom build). It follows `glutton`'s -wake/hold/hibernate lifecycle, but replaces synthetic HTTP `/ping` calls with a -real `"hello world"` process execution inside the guest (`StartProcess(["echo", -"hello world"])` + `StreamProcessOutput(follow: true)`) over gRPC directly -through `atenet-router` (`1 dns:///` gRPC channel per VU). - -#### ate-env Configuration Knobs - -Reuses `glutton`'s runtime and startup knobs: -`--min-wait-time`, `--max-wait-time`, `--min-live-time`, `--max-live-time`, -`--max-pings-per-wake`, `--lifecycle-mode` (`sync` | `async` | `none`), -`--trace-probability`, and `--actors-per-user`. - -#### ate-env Reported Metrics - -* `CreateAtespace`, `CreateActor`, `ResumeActorFirstResume`, `ResumeActor`, - `SuspendActor`, `SuspendActorAsync`, `DeleteActor`: Control-plane lifecycle - RPC latencies. -* `AteEnvExec`: Combined end-to-end latency of `"hello world"` (`StartProcess` + - `StreamProcessOutput` to exit with `exit_code == 0` and - `stdout == "hello world\n"`). -* `CrashCount`: Actors that entered `ACTOR_STATE_CRASHED` during resume. - -### Agent-Session Benchmark - -The agent-session benchmark (`--user-class agentsession`) emulates a fleet of -coding agents on Substrate. Each locust user is one session: an actor driven -through a scripted 20-step coding task ("clone a repo, build it, fix a test, -refactor, package it"), where every step costs the sandbox the CPU, memory, -disk, and network a real coding agent's action would. Between steps the agent -is "waiting for the LLM to think": the driver **suspends the actor** for the -step's think time, and the next step's first request **wakes it through the -atenet router** (request parking). Mostly-idle sessions plus fast wake is -exactly the oversubscription story this measures. - -The entire workload is a YAML script. The default, -[`internal/benchmarking/boomer/agentsession/scripts/coding-session.yaml`](../internal/benchmarking/boomer/agentsession/scripts/coding-session.yaml), -has one entry per step naming what the agent is doing and the resource ops -that act it out. To change the workload, edit it or add a sibling file and -select it with `--agentsession-script` (see [Writing a -script](#writing-an-agent-session-script)). Each session is an actor from -the stock `glutton` template; steps are sequences of glutton RPCs: - -| Step | The agent is… | Sandbox effect | -|---|---|---| -| 01_read_task | reading the task prompt | fill 32Mi RAM (agent context) | -| 02_clone_repo | `git clone` | 16Mi arrives over the network → disk; 0.5s CPU | -| 03_explore_tree | listing/grepping the tree | disk read (digest); 0.2s CPU | -| 04_read_key_files | opening files into context | disk read shipped back out; 8Mi RAM churn | -| 05_install_deps | `pip install` / `go mod download` | 32Mi over the network → disk; 1.5s CPU ×2 | -| 06_first_build | first full build | fill 64Mi RAM; 3s CPU ×2; 24Mi disk write | -| 07_run_unit_tests | running the test suite (one fails) | disk read; 2.5s CPU ×2 | -| 08_reason_about_failure | tracing the bug (long LLM turn, 8s think) | full RAM page-walk after the wake | -| 09_edit_source | applying the fix | 64Ki patch over the network; 4Mi disk write | -| 10_incremental_build | rebuilding changed packages | 1.2s CPU ×2; 8Mi disk write | -| 11_rerun_failed_test | re-running the failing test | 0.8s CPU | -| 12_write_new_tests | authoring regression tests (6s think) | 128Ki over the network; 2Mi disk write | -| 13_run_new_tests | running the new tests | disk read; 1s CPU | -| 14_full_test_suite | full-suite regression run | 4s CPU ×2; disk read; 8Mi RAM churn | -| 15_lint_format | lint + format pass | disk read; 0.9s CPU | -| 16_refactor | multi-file refactor (8s think) | 512Ki over the network; 6Mi disk write; 0.6s CPU | -| 17_rebuild | full rebuild | 32Mi RAM churn; 2s CPU ×2; 16Mi disk write | -| 18_final_test_suite | final full-suite run | 3.5s CPU ×2 | -| 19_package_artifact | building the release package | disk read; 1s CPU; 24Mi disk write | -| 20_commit_and_summarize | committing + summarizing | 256Ki disk write; 24Mi shipped back out; RAM walk | - -The script needs bigger actors than the 256Mi default: it holds ~96Mi of -RAM arrays + ~110Mi of tmpfs files, and the observed guest peak with -allocator transients is ~320Mi (512Mi OOMs). Deploy the workloads with -`--actor-memory 1Gi`. Every script declares that floor as -`min_actor_memory`; the worker reads the `glutton` template's memory limit -at start and refuses to run a script against a smaller actor, so a too-small -deployment fails loudly instead of showing up as OOM-flaky steps. - -```sh -./benchmarking/deploy_locust.sh --deploy --sandbox-class gvisor --actor-memory 1Gi -./benchmarking/locust/deploy.sh --deploy --user-class agentsession -``` - -#### Agent-Session Configuration Knobs - -* `--agentsession-script` — built-in script variant to run, by file name - under `internal/benchmarking/boomer/agentsession/scripts/` (default - `coding-session`). Resolved when a session starts, so a change takes - effect for sessions started after the next swarm; sessions already - running finish on the script they started with. -* `--agentsession-script-file` — path, on the boomer worker, of a script - YAML to run instead of a built-in variant; wins over - `--agentsession-script`. Normally set for you by - `locust/deploy.sh --agentsession-script FILE` (below). -* `--agentsession-think-scale` — multiplier on every think gap; 0.5 makes the - fleet twice as chatty, 4.0 models slow reasoning models (default 1.0). Each - gap gets ±20% jitter so sessions don't move in lockstep. -* `--resume-mode implicit|explicit` — implicit (default) lets the parked - first request wake the actor; explicit issues ResumeActor before traffic. -* `--lifecycle-mode sync|async|none` — synchronous suspend (`sync`, - uploads the snapshot before returning), asynchronous suspend (`async`, - default, uploads the snapshot in the background), or `none` to keep actors - resident in memory between steps without hibernating. -* `--agentsession-snapshot-steps` — comma-separated 1-based step numbers after - which the actor is hibernated (for example, `3,10,20,30,40,50,61`); between - all other steps the actor stays resident. -* `--agentsession-phase-in` — seconds over which session starts are spread: - each user waits a uniform random share of it before creating its actor - (default 0, every session starts at once). Locust's ramp spreads the fleet - over seconds and every session begins at step 1, so without it the - sessions reach each later step together; a step that dwells longer than - the ramp then asks every worker for more slots at once than it has. Set - it to a few think gaps for density runs. -* `--agentsession-resume-retry-budget` — seconds a step keeps retrying a - wake the fleet had no room for (ResumeActor `ResourceExhausted` or - `Unavailable`, router 503/504/429) with 50ms-to-1s backoff before giving - the step up until its next think gap (default 60; `0` gives the wake one - attempt). Each attempt is its own - `ResumeActor` or `WakeFirstTouch` sample; the whole wait lands in - `TimeToWake`. - -#### Writing an agent-session script - -A script is a named step list with a memory floor: - -```yaml -name: coding-session -min_actor_memory: 1Gi -steps: - - name: 01_read_task - agent: Boots, reads the task prompt, loads its context window - think: 2s - ops: - - fill_ram: {key: agent_context, size: 32Mi} - - ping: {} - - name: 02_clone_repo - agent: git clone of the target repository - think: 3s - ops: - - ingest: {key: repo_tarball, size: 16Mi} - - burn_cpu: {millis: 500, parallel: 1} -``` - -`think` is the suspended gap before the step (a Go duration, scaled by -`--agentsession-think-scale`); `name` keys the `Step_` stats row. -Sizes are Kubernetes quantities. Each op is one glutton RPC: - -| Op | Arguments | Sandbox effect | -|---|---|---| -| `ingest` | `key`, `size` | bytes cross the network from the driver and land in a file: a download | -| `burn_cpu` | `millis`, `parallel` (default 1) | spins `parallel` goroutines for `millis` of wall clock | -| `write_disk` | `key`, `size` | writes locally generated bytes to a file | -| `read_disk_digest` | `key` | reads and hashes a file; nothing ships back | -| `read_disk_data` | `key` | reads a file and returns its bytes: disk read plus egress | -| `fill_ram` | `key`, `size` | allocates a resident RAM array | -| `churn_ram` | `key`, `size` | re-randomizes part of an array in place, dirtying pages | -| `walk_ram` | `key` | touches one byte per page: demand-paging cost after a resume | -| `ping` | none | a minimal round trip through the router | -| `dwell` | `millis` | no request: the actor stays resident and idle, as a gateway does during a model round trip or a typing gap | - -Loading is strict: unknown op kinds or fields, an argument a kind does not -take, a read of a file nothing wrote, a walk of an array nothing filled, a -duplicate step name, or a `min_actor_memory` below the declared RAM plus -disk all fail before any actor is created. Built-in variants are checked by -`TestEmbeddedScriptsAreValid`, so a broken file cannot merge. - -#### Built-in script variants - -`coding-session` (default) is a coding agent working one task: 20 steps of -build, test and edit work, each preceded by a few seconds of LLM thinking -spent suspended; about 40 CPU-seconds of sandbox work per lap. - -`personal-assistant` is an always-on assistant of the OpenClaw or Hermes -Agent kind: one lap is one day in real time, with about 30 user messages -in 7 clusters, a heartbeat turn every 30 minutes while the user is awake, -hourly system-event crons, a morning briefing, model-catalog refreshes, -one context compaction and a nightly memory sweep (61 wakes, about 65 -CPU-seconds and 150 MiB written per lap, a 640Mi resident heap that grows -to about 960Mi). Each turn reads its session store, ships a 50 to 110 KiB -context out through the router, dwells for the model round trip, and -appends to its store, sized from an strace profile of OpenClaw; the -script's header comment lists the figures. Its think gaps are the real -idle gaps, so run it with a fractional think scale: -`--agentsession-script personal-assistant --agentsession-think-scale 0.02` -plays a day in about 30 minutes. Because every heartbeat and cron is a -wake, the script also shows what an always-on agent costs a system that -suspends it between events: an internal timer cannot fire in a suspended -sandbox, so each of those turns needs an external wake. - -`personal-assistant-light` is the same day with the RAM arrays scaled -down to 28Mi and everything else unchanged (files, reads, CPU, dwell and -the 61 wakes), for runs that want the always-on wake pattern with a small -memory image: about 40 to 95 MB per pause under gVisor, and under microVM -about 100 MB of guest before the script writes anything, bounded by the -actor's memory limit. It fits a 256Mi actor. Run it with the same -fractional think scale. - -To run a script of your own without rebuilding anything, hand it to the -locust deploy: - -```sh -./benchmarking/locust/deploy.sh --deploy --user-class agentsession --agentsession-script ./my-session.yaml -``` - -The script validates the file locally first (the same check the worker -runs, via `boomer-worker --check-agentsession-script`), uploads it as the -`agentsession-script` ConfigMap, mounts it into the boomer workers at -`/etc/agentsession/script.yaml`, and points the master's -`--agentsession-script-file` default at that path. Workers log the loaded -script's name, step count, and declared budgets on their first iteration. -To go back to a built-in variant, redeploy without the flag. - -#### Agent-Session Reported Metrics - -* `WakeFirstTouch`: latency of a dedicated ping sent before each step's ops — - in implicit mode that ping is what triggers the parked wake, so this row - **is** the user-visible wake latency, unpolluted by the step's own work. -* `TimeToWake`: the whole wake of a parked actor, from the first - `ResumeActor` attempt (in implicit mode, the first wake ping) to the ping - that found the actor serving, including every retry while the fleet had no - room. In implicit mode a first-attempt wake is the `WakeFirstTouch` sample; - in explicit mode it is `WakeFirstTouch` plus `ResumeActor`. A failed row is - a wake that failed: a step given up after - `--agentsession-resume-retry-budget`, or one that failed at once with an - error that is not a capacity error, such as a crashed actor or a router - 404 or 500. -* `Step_` (e.g. `Step_06_first_build`): wall time of that step's ops, - think gap excluded. -* `SuspendActor` / `ResumeActor` / `CreateActor` / `DeleteActor`: control-plane - lifecycle latencies. - -### Spawn Benchmark - -The Spawn benchmark creates a batch of actors once and measures how long each -actor takes from creation to its first answered ping, and how long the whole -batch takes. `tests.yaml` runs it as `spawn_smoke_10_actors` with -`shapes/spawn_shape.py`, which holds one user and ends the run once -`TimeToAllReady` is recorded. - -Each boomer process creates one batch; extra users in the same process do -nothing. Actors are named `spawn--` and deleted when boomer exits. - -#### Spawn Configuration Knobs - -* `--total-actors`: Actors in the batch (default `100`). -* `--spawn-concurrency`: Actors created concurrently (default `1`). -* `--actor-deadline`: Per-actor timeout in seconds, covering create, resume and - first ping (default `120`). - -The web UI shows the same fields; `0` keeps the value boomer-worker started with. - -#### Spawn Reported Metrics - -* `CreateActor`, `ResumeActor`, `GluttonPing`: Latency of each call. -* `ActorTimeToReady`: Per actor, from its first `CreateActor` attempt to its - first successful ping. -* `TimeToReady_pct` (`k` = 10, 20, … 100): From batch start until `k`% of - the batch was ready. -* `TimeToAllReady`: From batch start until the last ready actor answered. - Actors that failed show up as `ActorTimeToReady` failures instead. -* `CrashCount`: Actors that crashed during resume. - -The `actors_per_*` ratios in `trial_summary` are wrong for this test: they -count users × `--actors-per-user`, not `--total-actors`. - -### ResumePaused Benchmark - -The ResumePaused benchmark drives ate-api-server's cold resume path at a set -rate. A cold resume here is a ResumeActor of a suspended actor: ateapi takes the -actor lease, binds a worker, and restores the actor from its snapshot. A -ResumeActor of a running actor does none of that. Each user owns one actor -from the `glutton` template. On its first iteration the user creates the actor -and makes it cold with a resume and a suspend; every later iteration resumes the -actor and suspends it again, so every measured resume is cold. - -The warm-up runs in the step that added the user, unpaced, and is recorded -under its own names. - -It sends no requests to the actors themselves, so it runs on the fake data -plane, where it measures what a cold resume costs ate-api-server and Postgres -without the router or the restore in the way. The suspends run at the resume -rate, so a capacity it finds is one of resume and suspend cycles, not of resumes -alone. - -Actors are named `rp-` and deleted when boomer exits. If the atespace -cannot be ensured, boomer exits at startup. An actor that fails to warm is -deleted, and its user creates another on its next iteration. When a step lowers -the user count, the actors of the stopped users stay suspended, unused, until -boomer exits. - -#### ResumePaused Configuration Knobs - -* `--target-rps`: Cold resumes per second the whole boomer process paces to - (default `0`, unpaced). A `--ladder` step ending in `@target_rps`, as in - `200:3m@1000`, changes it as the step starts. Keep the users at or above the - rate times the p99 of one resume plus one suspend in seconds, or the step falls - short of its rate. - -The web UI shows the same field. - -Trim the start of each ladder step. A rate change reaches boomer at its next -poll of the master, within 10 s, and a step that adds users carries their -warm-up until they have all joined at the spawn rate. - -#### ResumePaused Reported Metrics - -* `ResumeActorCold`: Latency of each measured resume. A resume that finds its - actor already running fails, as no cold resume happened. -* `SuspendActorAsync`: Latency of the async SuspendActor that makes the actor - cold again. -* `SuspendActorAsyncRecovery`: After a failed cycle, the async SuspendActor that - makes the actor cold for the next one. It is load on the server, so it is - recorded, but apart from `SuspendActorAsync`. A recovery suspend that finds - the actor already suspended counts as a success. -* `CreateActor`, `ResumeActorWarmup`, `SuspendActorAsyncWarmup`: A user's - warm-up of its actor, in the step that added the user. - -#### Analyzing a ResumePaused Ladder Run - -`stats.csv` and the other rows of `stats.jsonl` cover a whole run, so they -blend every step of a ladder. After a ladder run the runner adds one -`stats.jsonl` row per step, with `metric` `ladder_step_001`, -`ladder_step_002`, and so on ([`locust/ladder_steps.py`](locust/ladder_steps.py)). -It reads each step's start and end from the run's log and queries the -benchmarking Prometheus over the step's steady window, from 30 s after the -step starts to 5 s before it ends (`window_start` and `window_end`). A row -holds the cold resume and pause rates, failures by operation -(`failures_`), client and server latency, the CPU and memory of -ate-api-server, Postgres and fake-atelet, and Postgres's connections, -commits and waits. - -The runner logs and skips a step with no end in the log and a step too short -to leave a window after the trim. A failed query adds no step rows at all -rather than a ladder with gaps, and leaves the run's other results as they -are. Read a step's figures against its `target_rps`: a `resume_rps` below it -means the step fell short of its rate. - -To pull the step rows out of a run's results: - -```bash -RUN=/runs//run_date=/run_ts=/run_tag= -gcloud storage cat "$RUN/stats.jsonl" | grep '"metric": "ladder_step_' -``` - -`` is the orchestrator's `--dest`, and its log has the full `RUN` path -on each `Uploaded ... -> gs://...` line. - -### Isolate Mode - -A real cold resume is limited by the node's restore capacity, which hides the -limits of ate-api-server and Postgres. Isolate mode takes the nodes out by -installing substrate with a fake data plane: - -```bash -hack/install-ate.sh deploy ate-system --fake-data-plane -./benchmarking/workloads/deploy.sh --deploy --worker-count 200 \ - --fake-atelet-env FAKE_ATELET_DELAY=500ms -``` - -`deploy.sh` deploys the usual WorkerPool, but the install serves it with two -fakes from `cmd/benchmarking/isolate/` instead of worker pods: - -* `fake-workersync` stands in for ate-controller's WorkerPool controller and - worker syncer. It honors each pool's `spec.replicas` with fake Workers, none - backed by a pod, round robin on the nodes that run fake-atelet, and scales them - up and down with the pool, draining a Worker before deleting it. Each Worker - reports what a real worker would: cpu and memory from the pool's - `template.resources.limits`, or the node's allocatable without one, and 1000 - actors, ateworker's default, unless the pool's `ate.dev/fake-max-actors` - annotation says otherwise. It writes the pool's status, so - `kubectl get workerpool` and `kubectl scale workerpool` work. A node keeps - its Workers for `--node-grace`, five minutes by default, after its - fake-atelet pod goes, so a DaemonSet rollout does not move them; past that - they move to a node that still runs fake-atelet. -* `fake-atelet` takes atelet's place in its DaemonSet, on every node atelet - would run on, and answers every - lifecycle call with success after its `--delay`, without running or - saving any workload. It writes one placeholder object per snapshot, because - ate-api-server refuses to tag an empty one. Like ateworker, it mints the actor's - certificate from ate-api-server on every run and restore that names an - egress gateway, which the default install does; the mint is ate-api-server's - time, so it adds to `--delay`, the data plane's share of the call. - It also relays fake-workersync's capacity reports, as atelet relays ateworker's: - ate-api-server accepts a report only from the atelet on the Worker's node. - The relay admits only fake-workersync's pod identity. fake-atelet remembers - the epoch each Worker registers with and reports it from every run and - restore, as a worker process does; ate-api-server commits a resumed actor - only under its Worker's current epoch. It keeps the epochs in memory only, - and fake-workersync registers a Worker once, so a fake-atelet that restarts - after its Workers registered, in a DaemonSet rollout for example, fails - every resume on those Workers with Unavailable until they are replaced, - such as by redeploying the benchmark. - -The install also scales ate-controller to zero, whose worker syncer would -delete the fake Workers. Its other controllers stop with it: no NetworkPolicy -is made for a WorkerPool, and the egress MITM trust bundle is not derived. -Fake actors need neither. fake-atelet takes the storage settings of atelet's -objectstore-plugin sidecar, whose writes it takes over, so its snapshot -placeholders go to the storage the install points atelet at. The install -refuses a cluster whose WorkerPools still have worker pods, which would -outlive ate-controller. A normal `hack/install-ate.sh deploy ate-system` -puts the real data plane back, and `delete ate-system` removes the fakes -either way. Never use the mode on a -cluster that serves real actors: actors placed on fake Workers do not exist. - -The installer only chooses the fake data plane; the benchmark tunes it. -fake-atelet reads each tuning flag from a `FAKE_ATELET_` variable in the -optional `fake-atelet-tuning` ConfigMap, which no install creates or resets: -`FAKE_ATELET_DELAY` sets `--delay`, `FAKE_ATELET_DELAY_RESTORE` overrides it -for Restore alone, and `FAKE_ATELET_MINT_ACTOR_CERTIFICATE=false` skips the -certificate mint. `deploy.sh --fake-atelet-env NAME=VALUE`, repeatable, writes -that ConfigMap to exactly the given entries and restarts fake-atelet when it -changes, before the WorkerPool is applied. - -Nothing answers behind the router, so the mode suits only a user class that -sends the actors no requests, such as `ate_api.py`, which creates, gets, -resumes and suspends actors. Every boomer user class pings its actors through -the router. In `tests.yaml`, an `isolate` block selects the mode, and its -keys become `deploy.sh --fake-atelet-env` entries: `delay` sets -`FAKE_ATELET_DELAY`, and so on. - -### Viewing Traces -You must have enabled otel tracing for your cluster to view traces. - -You can find trace IDs by viewing the `logs` tab in the Locust UI - -## Benchmark output files - -A run writes the following to `--dest`. Each run produces them fresh; none of -them are checked into the repository. - -* `status.json`: `locust_exit_code` and `stats_generated`. Deliberately just - those two keys, because it is what CI orchestration reads to decide whether a - trial ran at all. -* `stats.csv`, `stats_history.csv`, `failures.csv`, `exceptions.csv`: Locust's - own CSV output. `stats_history.csv` has a row per operation every second as - well as the `Aggregated` row, so each operation's latency can be charted - over the run; filter to `Aggregated` for run-wide numbers. -* `logs.txt`, `traces.txt`: the runner log, and the trace IDs seen during the run. -* `stats.jsonl`: one JSON object per line, one per metric. Every row carries - the same five keys: `timestamp`, `tag`, `test_name`, `metric`, and a flat - `measurements` map holding that metric's numbers. The `phase_*` rows and - the `phase_breakdown_summary` row are the suspend/resume phase percentiles - read from the node logs (see - [analysis/README.md](analysis/README.md#automated-runs)). -* `server_summary.json`: server-side telemetry harvested from Prometheus, - including the per-sample bin-packing timeseries. - -### Density frontiers - -With cluster discovery enabled, `stats.jsonl` gains a `trial_summary` row -describing how densely actors are packed onto the hardware. Its `measurements` -map holds the raw facts and the derived numbers side by side. - -* `machine_type`, `node_count`, `allocatable_cores`, `allocatable_ram_gb` - (GiB), `worker_pod_count`: the measured facts, before any arithmetic. - Capacity covers the nodes the worker pods are running on rather than the - whole cluster, so a separate infrastructure pool is not counted. They are - recorded so the ratios below can be re-derived later, or recomputed against - a different denominator. -* `kubernetes_version`: the worker nodes' Kubernetes version (kubelet version). -* `worker_actor_capacity`, `worker_memory_limit_gb` (GiB): the capacity each - worker pod declares, its ateworker `--max-actors` (1000 when unset) and its - container memory limit (null when unset, as the node bounds it instead). - `sandbox_class`: `gvisor` or `microvm`, from the worker image. A mixed pool - gives a sorted comma-joined list for each. An unreadable value is skipped - without affecting the others, and logged once. -* `multi_actor_worker`: `true` when `actors_per_pod_p50` is above 1. -* `run_start`, `run_end`: Unix seconds bracketing the run, for finding its - server-side metrics after the cluster is deleted. -* `metadata_`: one per key of the free-form `--metadata ''` object, - for facts only the caller knows. Non-string values are stored as JSON, and - null values are skipped. The orchestrator passes `cluster_name`, - `cluster_location` and `project_id` from the target cluster, plus any keys in - the `BENCHMARK_METADATA` environment variable (a JSON object) and in the - test's own `--metadata` flag, whose keys win on conflict. -* `actors_per_node`, `actors_per_vcpu`, `actors_per_gb_ram`: the most actors - Locust reported running, over the matching capacity. The `-u` flag only - stands in when no sample was read. -* `actors_per_pod_p50`, `actors_per_pod_p90`, `actors_per_pod_p99`: actors per - worker pod across the run. Reported as a distribution rather than one - average, and it spans ramp-up too, because a custom load shape has no - single user count to call steady. -* `aggregate_failure_ratio`: failures over requests for the run. -* `_failure_ratio`: the same ratio for every operation Locust - reported, so each test carries its own names through. The operation name is - lowercased with underscores, so `DurDirWrite` becomes - `dur_dir_write_failure_ratio`. A key is absent when the test has no such - row, and null when the row ran no requests. - -The six `actors_per_*` ratios rest on three assumptions. Read them before -comparing numbers across runs: - -* **Actors are derived, not counted.** Locust only sees virtual users, so the - numerator is the peak user count times `--actors-per-user`. No server-side - gauge counts resident actors: `ate.actor.stats.sampled_actors` drops any - actor without a live resource measurement, so suspended ones fall out. It - is recorded as `active_actors` in `server_summary.json`, a running-actor - count rather than a resident one. -* **The denominators are read once, after the run.** A cluster that autoscaled - mid-run is measured at its final size, so the ratio pairs a peak from one - moment with a capacity from another. The same holds for every worker and - node fact above, including `kubernetes_version` and the worker capacity, and - so for `multi_actor_worker`, which is derived from `actors_per_pod`. -* **The peak assumes every actor is alive at once.** A workload that creates - and deletes actors as it goes never holds them all at the same time, so its - real density is lower than reported. - -### Server ground truth - -With a reachable Prometheus, `server_summary.json` records what the server -actually did, independent of what the load generator reported. - -* `cluster_packing`: workers holding at least one actor (`partial` or - `at_capacity`) over the pool size at each sample (the sum of all - `ate_workerpool_workers` states), as a percentile `summary` plus the - per-sample `timeseries` it was computed from. -* `node_psi.cpu_stall_pct`, `mem_stall_pct`, `io_stall_pct`: kernel pressure - stall percentages on the nodes hosting a worker pod, with every node's - samples pooled. `node_psi.nodes` is how many nodes were seen. -* `pod_psi.cpu_stall_pct`, `mem_stall_pct`, `io_stall_pct`: the same for each - worker pod's cgroup slice, which also holds its gVisor sandbox, with every - pod's samples pooled. `pod_psi.pods` is how many pods were seen. -* `snapshots.size_p50_mb` through `size_p99_mb`: actor memory image sizes. -* `snapshots.size_avg_mb`: mean memory image size. -* `snapshots.restore_p50_s` through `restore_p99_s`, `restore_mean_s`, and the - same for `checkpoint_*`: how long the atelet took to resume or suspend an - actor (`ate_actor_{restore,checkpoint}_duration`, `total` phase). Every - snapshot kind counts, as in the sizes: the actor's own snapshot and a - template's first start from its golden snapshot. The buckets reach 60s, so a - slower operation reads as 60s. -* `snapshots.checkpoints_in_window`, `checkpoints_cumulative`: checkpoint - volume over the steady-state window, and since the atelet started. -* `snapshots.checkpoint_mb_s`: bytes written per second spent checkpointing, - not per second of wall clock. -* `active_actors`: running actors (`ate_actor_stats_sampled_actors`), as a - percentile `summary` of the cluster-wide count, `atelets` seen, and a - `timeseries` every 10s over the whole run (ramp-up included). Each point has - the cluster-wide count and `active_atelets` hosting an actor. As the - assumptions above note, the gauge drops actors without a live measurement, - so suspended ones fall out and the count dips while actors are suspended. - The atelet stops exporting when a node has no running actor, so at a step - where at least one atelet reported, a missing one counts as 0. A step where - none reported is skipped, not 0, since a failed scrape looks the same; this - mostly hits 1-user runs. If the metric never appeared during the run, every - field is `null`. `atelets` counts atelet processes seen in the run, not - nodes: the collector keeps an exited atelet's last value for about 5 - minutes, so a restart can briefly count twice, and an atelet scraped both - directly and through a collector counts twice. -* `working_set.node`, `worker`, `atelet`: memory working set in GiB from - cAdvisor on the nodes hosting a worker pod, per worker pod (every actor on it - included), and per atelet on those nodes. Each has a percentile `summary` - (plus `max`) over the steady-state window, `count` of series seen in it, and - a `timeseries` every 10s over the whole run with the sum (`total_gb`) and - largest (`max_gb`). A spike shorter than 10s can be missed. -* `working_set.actor`, `per_actor`: the same shape for the actors' own working - set as the atelet reports it (`ate_actor_stats_memory_working_set_bytes`), - per atelet with every template summed, and the average per measured actor - (`ate_actor_stats_sampled_actors`) on each atelet; `count` is atelet - processes, keyed like `active_actors`. As `per_actor` is an average per - atelet, its `max` is the highest atelet average, not the largest actor, and - its `total_gb` has no meaning. A gVisor actor is read from its sandbox cgroup - on the host, a microVM actor from inside the guest. As with `active_actors`, - suspended actors fall out, but through the telemetry-meter a series that - stops being exported keeps its last value for up to 5m. Missing samples are - skipped, not counted as 0. The atelet samples every - `--actor-stats-poll-interval` (default 1m), so these move in steps of a - minute or more. `actor` matches one worker pod's actors only when the node - runs one worker pod. -* `oom_events.node`, `pod`: OOM kills over the run, from cAdvisor's - `container_oom_events_total`. `node` counts only global OOMs. `pod` counts - kills inside a worker pod (actor sandboxes and the worker container) and is - best effort: a non-zero value is real, but a kill whose cgroup is gone before - the scrape is missed. A node-allocatable OOM (`oom_memcg=/kubepods.slice`) - isn't global, so it isn't in `node`, and `pod` usually misses it; the kubelet - evictions that often come with it show in `evicted`. OOMs inside a microVM - guest are invisible and show up as task failures. Memory PSI (`node_psi` / - `pod_psi`) is the early warning. -* `oom_events.worker_restarts`, `worker_oomkilled_pods`, `evicted`, - `worker_pods_lost`: from the Kubernetes API, read before and after the run. - Restarts on a pod deleted during the run are lost. `worker_oomkilled_pods` - counts pods whose last restart was `OOMKilled`, not kills; a group kill of - the whole worker (`memory.oom.group=1`) shows here. `evicted` is kubelet - eviction under node pressure (usually memory), not a kernel OOM kill. It also - counts replacement pods the kubelet refused to start while the node was under - pressure, as they get the same `Evicted` reason. It comes from `Evicted` - events, which last about 1h, so a run longer than that can undercount. - `worker_pods_lost` is pods gone by the end for any reason (eviction, - scale-down, recycling), not an OOM count; a pod created and deleted within - the run isn't counted. A node OOM can also show as a worker `OOMKilled` - restart, so never add the fields. A field is `null` when its Kubernetes read - fails, and all are with `--no-cluster-facts`. - -Every distribution reports p50, p90, p95 and p99 over the steady-state window. -The steady-state window runs from the first to the last Locust sample at 90% or -more of the run's peak user count, so ramp-up and teardown are left out. Under -a step-ladder load shape it covers only the top step. -Counts, means and `checkpoint_mb_s` come from the growth of the histogram's sum -and count across the window, and the size, restore and checkpoint percentiles -are interpolated inside its buckets. PSI is a one-minute rolling average read -every 10s. - -The atelet exports before Prometheus scrapes it, so the harvest waits -`--atelet-lag-s` seconds (default 70, enough for the OTel SDK's 60s default -export and a 10s scrape) and reads the snapshot, `active_actors` and actor -working set windows half that late, stamping `active_actors` and actor -working set points back by the same amount. The window ends at the last -full-load sample, so teardown suspends are left out. - -`metadata.start_ts` and `end_ts` bound the whole run, which the packing, -`active_actors` and `working_set` timeseries cover. `steady_start_ts` and -`steady_end_ts` bound the steady-state window. A flat subset of the numbers -also goes into a `server_summary` row in `stats.jsonl`. - -Neither the Kubernetes API nor Prometheus is required. If either is unreachable, -or discovery was skipped, the affected fields are written as `null` and the run -still succeeds. A `null` means the value was not measured. It never means zero. - -## Suspend/resume phase breakdown - -`SuspendActor` and `ResumeActor` latency can be attributed to the phases -inside them (object-storage transfer, hypervisor snapshot/restore, rootfs -assembly, ...) from the structured log records atelet and ateworker-microvm -emit. `analysis/collect_logs.sh` dumps the node logs of a run and -`analysis/phase_report.py` aggregates them into per-phase percentiles and -waterfalls of the slowest operations. See -[analysis/README.md](analysis/README.md). - -## Optional: Prometheus + Grafana - -Locust provides graphs, statistics, etc. via the UI. However, you -can install Prometheus/Grafana if you want richer details or -the ability to perform deeper analysis. Skip this section if -you're only using the Locust web UI. - -```bash -kubectl apply -f benchmarking/monitoring.yaml -``` - -Once installed: - -* Run `kubectl port-forward svc/grafana -n benchmarking 3000:3000` -* Visit `http://localhost:3000` in your browser. - -## Development - -### Generating gRPC Python clients - -The clients are not checked in. The locust and nighthawk-ingress images -generate them at build time, and `hack/verify/python-protos.sh` compiles them -on every PR, so a proto change needs no extra step. For local use, such as -editor completion, run `benchmarking/locust/codegen/generate.sh`. It manages -its own virtual environment under `locust/codegen/venv`. - -### Unit tests - -`locust/unit_tests` covers the runner's helpers and needs no cluster. From the -repository root: - -```bash -python3 -m unittest discover -s benchmarking/locust/unit_tests -``` - -`analysis/test_phase_report.py` covers the phase-log report's parser and -aggregation: - -```bash -python3 -m unittest discover -s benchmarking/analysis -``` +Moved to [docs/guides/benchmarking.md](../docs/guides/benchmarking.md). diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 7f36ddf5be..ad54f51d67 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -7,6 +7,8 @@ Each run uploads results to GCS via `benchmarking/locust/runner.py` (`type: locust`) or `benchmarking/nighthawk-ingress/runner.py` (`type: nighthawk-ingress`, the router capacity benchmark — see [benchmarking/nighthawk-ingress/README.md](../nighthawk-ingress/README.md)). +To run the cold resume tests from your workstation instead, see +[ateapi-cold-resume.md](ateapi-cold-resume.md). ## How it works @@ -18,14 +20,14 @@ the router capacity benchmark — see 4. `docker build && docker push` builds the runner image for each test type in use, tagged with the commit hash: `${KO_DOCKER_REPO}/locust-test:` and/or `${KO_DOCKER_REPO}/nighthawk-ingress-test:`. -5. `hack/install-ate.sh deploy ate-system` (with `ATE_CREDENTIAL_PROVIDER` +5. `go run ./cmd/ate-setup deploy ate-system` (with `ATE_CREDENTIAL_PROVIDER` in `.ate-dev-env.sh`, or `--credential-provider` in the test's `ateArgs`) + `benchmarking/workloads/deploy.sh --deploy --sandbox-class ` (these build & push substrate / workload images via `ko` as part of their deploy steps — there's no separate image build step). The install sets `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` to the `ate-client` and `benchmarking/benchmark-runner` ServiceAccounts so the - runner's calls pass [authorization](../../docs/authorization.md), and sets + runner's calls pass [authorization](../../docs/guides/authorization.md), and sets `ATE_INGRESS_ALLOWED_SPIFFE_IDS` to `benchmarking/benchmark-runner` for the Envoy ingress router. For a `microvm` test the orchestrator also runs `hack/install-microvm-deps.sh --install` between the two, which @@ -147,7 +149,7 @@ principal://iam.googleapis.com/projects//locations/global/worklo - `roles/container.admin` on the test cluster's project — required to manage cluster-scoped resources (CRDs, ClusterRoles, ClusterRoleBindings, - Namespaces) that `hack/install-ate.sh deploy ate-system` creates. + Namespaces) that `ate-setup deploy ate-system` creates. `container.developer` is not enough — it intentionally omits the `container.clusterRoles.*` and `container.customResourceDefinitions.*` permissions. diff --git a/benchmarking/automation/ateapi-cold-resume.md b/benchmarking/automation/ateapi-cold-resume.md new file mode 100644 index 0000000000..805d2383c2 --- /dev/null +++ b/benchmarking/automation/ateapi-cold-resume.md @@ -0,0 +1,153 @@ +# Running the ate-api-server cold resume test locally + +The `ateapi_cold_resume_pause_fake_*` tests in [tests.yaml](tests.yaml) +measure how many cold resumes per second ate-api-server and Postgres sustain. +They run the [ResumePaused benchmark](../README.md#resumepaused-benchmark) on +the [fake data plane](../README.md#isolate-mode) as a rate ladder. + +Both tests run through `benchmarking/automation/orchestrator.py`. The +[automation README](README.md#how-it-works) covers what it does for each test +and how to run it from the CronJob. This doc covers a one-off run from your +workstation. Run the commands from the repository root. + +## 1. Cluster + +Create the test cluster as in +[Test cluster prerequisites](README.md#test-cluster-prerequisites). + +Both tests pass `--cordon-system-infra-components`, which needs the two node +pools in [System Infra Isolation](../../docs/architecture.md#system-infra-isolation): +`ate-system-infra` and `ate-postgres`. If the `ate-postgres` pool is missing, +the install fails before it applies Postgres. If the `ate-system-infra` pool +is missing or too small, its pods stay Pending and the install fails when +the rollout wait times out. Size that pool for ate-api-server's +`replicas × cpu` plus the other system infra pods. Both tests run one +ate-api-server at 2 CPUs. + +```bash +gcloud container node-pools create ate-system-infra \ + --cluster=$CLUSTER_NAME --location=$CLUSTER_LOCATION \ + --machine-type= --num-nodes=1 \ + --node-labels=ate.dev/workloadType=ate-system-infra \ + --node-taints=ate.dev/workloadType=ate-system-infra:NoSchedule + +gcloud container node-pools create ate-postgres \ + --cluster=$CLUSTER_NAME --location=$CLUSTER_LOCATION \ + --machine-type= --num-nodes=1 \ + --node-labels=ate.dev/workloadType=ate-postgres \ + --node-taints=ate.dev/workloadType=ate-postgres:NoSchedule +``` + +The default pool runs the locust runner (8 CPUs in `_2cpu`) and fake-atelet. + +## 2. Prometheus + +The orchestrator does not deploy Prometheus. The runner needs it for +`server_summary.json` and the per-step rows in `stats.jsonl`, so install it +once as in [Prometheus + Grafana](../README.md#optional-prometheus--grafana). + +## 3. Workstation + +You need `gcloud`, `kubectl`, `docker`, Go, and Python 3 with PyYAML. ko +runs from `hack/tools/ko` through `go tool`, so it need not be installed. +The orchestrator polls `docker info` first and fails if Docker is not +running within 2 minutes. + +```bash +python3 -m venv benchmarking/automation/venv +benchmarking/automation/venv/bin/pip install -r benchmarking/automation/requirements.txt +gcloud auth login +gcloud auth configure-docker gcr.io +``` + +Your account needs the orchestrator's roles, and the runner's KSA needs +write access to the results bucket. See +[IAM prerequisites](README.md#iam-prerequisites). + +## 4. Pick the tests + +The orchestrator runs every test in the file you pass to `--tests`, so copy +the entries you want into a new file: + +```bash +python3 - <<'PY' > /tmp/cold-tests.yaml +import yaml +want = {"ateapi_cold_resume_pause_fake_smoke"} +t = yaml.safe_load(open("benchmarking/automation/tests.yaml"))["tests"] +print(yaml.safe_dump({"tests": [e for e in t if e["name"] in want]}, sort_keys=False)) +PY +``` + +| Test | Load | Duration | +|---|---|---| +| `ateapi_cold_resume_pause_fake_smoke` | 50 users, 20 then 50 cold resumes/s | 6m | +| `ateapi_cold_resume_pause_fake_2cpu` | 1000 users, 100 to 1500 cold resumes/s in 7 steps | 23m | + +Run `_smoke` first to check the setup before the 23 minute ladder. The +header comments in `tests.yaml` describe the fields, such as +`controlPlane`, `isolate` and `--ladder`. If you change `replicas` or +`poolMaxConns`, keep within Postgres's connection slots as described in +[Running at scale](../README.md#running-at-scale). + +## 5. Target cluster file + +The orchestrator copies `.sh` from the `--target-cluster-dir` +directory into the clone as `.ate-dev-env.sh`. Both tests use +`targetCluster: dev`, so step 6 copies your `.ate-dev-env.sh` there as +`dev.sh`. Create it from +[hack/ate-dev-env.sh.example](../../hack/ate-dev-env.sh.example) +if you don't have one. The install needs `PROJECT_ID`, `CLUSTER_NAME`, +`CLUSTER_LOCATION`, `KO_DOCKER_REPO`, `SNAPSHOT_LOCATION`, +`SNAPSHOT_TRANSFER_GSA_NAME` and `ATE_CREDENTIAL_PROVIDER`. + +## 6. Run + +The orchestrator clones `--branch` from `--repo`, so commit your changes +first. A `file://` URL clones your local checkout. Set `SUBSTRATE_DIR` to a +directory that does not exist yet, because the default is the pod path +`/workspace/substrate`. The orchestrator runs `gcloud container clusters +get-credentials`, so point `KUBECONFIG` at a separate file to keep your +current context: + +```bash +S=$PWD # this checkout +D=$HOME/bench-runs/cold-$(date +%Y%m%d-%H%M) +mkdir -p $D/target-clusters +cp $S/.ate-dev-env.sh $D/target-clusters/dev.sh +export KUBECONFIG=$D/kubeconfig + +cat > $D/run.py </ \ + --tests /tmp/cold-tests.yaml \ + --target-cluster-dir $D/target-clusters \ + --manifests-dir $S/benchmarking/automation/manifests \ + --junit-output $D/junit.xml 2>&1 | tee $D/orchestrator.log +``` + +Keep your workstation awake for the whole run. If it suspends, the +orchestrator pauses too, and the teardown and the next test wait for it. For +long ladders, run the command under `setsid nohup` or in a VM. + +## 7. Results + +[Analyzing a ResumePaused Ladder Run](../README.md#analyzing-a-resumepaused-ladder-run) +explains the per-step rows and where the run path is in the log. +[Benchmark output files](../README.md#benchmark-output-files) +lists the files. To copy them all down: + +```bash +st=$(grep -o 'gs://[^ ]*/status.json' $D/orchestrator.log | head -1) +mkdir -p $D/results && gcloud storage cp "${st%/status.json}/*" $D/results/ +``` + +The capacity is the highest step whose `resume_rps` keeps up with its +`target_rps` and has no failures. diff --git a/benchmarking/automation/controlplane.py b/benchmarking/automation/controlplane.py index 1b4003913d..75f0c964d0 100644 --- a/benchmarking/automation/controlplane.py +++ b/benchmarking/automation/controlplane.py @@ -17,8 +17,9 @@ (see API_SERVER_KNOBS) and a `postgres:` section (see POSTGRES_DEFAULTS). A test type validates the block with validate() and calls shape() from its -pre_test. Isolation from the load is the install's: --cordon-control-plane -in ateArgs keeps ate-api-server and Postgres on their own tainted pools. +pre_test. Isolation from the load is the install's: +--cordon-system-infra-components in ateArgs keeps ate-api-server and Postgres +on their own tainted pools. """ import json diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index 76f3272e28..c50fdc2946 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -126,7 +126,7 @@ def apply_target_cluster( target_cluster: str, target_cluster_dir: str = TARGET_CLUSTER_DIR ) -> None: """Copy target_cluster_dir/.sh into the cloned - substrate repo as .ate-dev-env.sh (so install-ate.sh / deploy.sh source + substrate repo as .ate-dev-env.sh (so ate-setup / deploy.sh source it) and merge it into this process's env (so the orchestrator's own gcloud / docker / kubectl calls see the same values). Resets os.environ to the startup baseline first so vars defined by a previous target @@ -390,7 +390,7 @@ def validate_and_normalize_tests(tests: list[dict[str, Any]]) -> None: def substrate_args(test: dict[str, Any]) -> list[str]: - """Return the hack/install-ate.sh deploy ate-system arguments for a + """Return the ate-setup deploy ate-system arguments for a tests.yaml entry: its ateArgs, then the fake data plane flag its isolate block asks for. The teardown needs no matching flag: delete ate-system removes the fake data plane whether or not it was installed.""" @@ -413,11 +413,11 @@ def deploy_substrate(ate_args: Iterable[str] = ()) -> None: "ATE_API_AUTHZ_BOOTSTRAP_OWNERS": f"{CLIENT_PRINCIPAL},{BENCHMARK_RUNNER_PRINCIPAL}", "ATE_INGRESS_ALLOWED_SPIFFE_IDS": BENCHMARK_RUNNER_PRINCIPAL, } - run(["hack/install-ate.sh", "deploy", "ate-system", *(str(a) for a in ate_args)], env=env) + run(["go", "run", "./cmd/ate-setup", "deploy", "ate-system", *(str(a) for a in ate_args)], env=env) def teardown_substrate() -> None: - run_no_check(["hack/install-ate.sh", "delete", "ate-system"]) + run_no_check(["go", "run", "./cmd/ate-setup", "delete", "ate-system"]) def install_microvm_deps() -> None: @@ -451,8 +451,8 @@ def deploy_workloads( "--sandbox-class", sandbox_class, ] - # Empty keeps the default in workloads/deploy.sh (256Mi, the microvm - # minimum); RAM-consuming suites set actorMemory in tests.yaml. + # Empty keeps the default in workloads/deploy.sh (256Mi); RAM-consuming + # suites set actorMemory in tests.yaml. if actor_memory: cmd += ["--actor-memory", actor_memory] # Empty keeps the default in workloads/deploy.sh (1 CPU). The scheduler @@ -595,7 +595,7 @@ def main() -> None: # Per-target-cluster caches: re-running setup for the same target # cluster is wasted work, so we track what was last set up and only # redo it when the target cluster name changes (substrate images get - # rebuilt by install-ate.sh each test anyway via ko apply). Runner + # rebuilt by ate-setup each test anyway via ko). Runner # images are built lazily per test type, so a locust-only tests.yaml # never builds the nighthawk image and vice versa. last_target = None diff --git a/benchmarking/automation/test_orchestrator.py b/benchmarking/automation/test_orchestrator.py index 77833d43ba..3cf8f809b8 100644 --- a/benchmarking/automation/test_orchestrator.py +++ b/benchmarking/automation/test_orchestrator.py @@ -68,7 +68,7 @@ class DeploySubstrateTest(unittest.TestCase): def test_runner_and_client_are_bootstrap_owners(self, run): orchestrator.deploy_substrate(["--credential-provider={}"]) cmd = run.call_args.args[0] - self.assertEqual(cmd, ["hack/install-ate.sh", "deploy", "ate-system", "--credential-provider={}"]) + self.assertEqual(cmd, ["go", "run", "./cmd/ate-setup", "deploy", "ate-system", "--credential-provider={}"]) env = run.call_args.kwargs["env"] owners = env["ATE_API_AUTHZ_BOOTSTRAP_OWNERS"].split(",") self.assertEqual( @@ -92,7 +92,7 @@ def test_keeps_the_rest_of_the_environment(self, run): @mock.patch("orchestrator.run_no_check") def test_teardown_substrate(self, run_no_check): orchestrator.teardown_substrate() - run_no_check.assert_called_once_with(["hack/install-ate.sh", "delete", "ate-system"]) + run_no_check.assert_called_once_with(["go", "run", "./cmd/ate-setup", "delete", "ate-system"]) class DeployWorkloadsTest(unittest.TestCase): diff --git a/benchmarking/automation/tests.yaml b/benchmarking/automation/tests.yaml index 678cbf6ec5..1a5eae67ef 100644 --- a/benchmarking/automation/tests.yaml +++ b/benchmarking/automation/tests.yaml @@ -32,8 +32,8 @@ # deploy that runs before this test. The orchestrator # redeploys workloads between tests, so each test can # pick its own scale. -# actorMemory Optional (default "256Mi", the smallest size microvm -# admits). Memory limit on the benchmark ActorTemplates; +# actorMemory Optional (default "256Mi"). Memory limit on the +# benchmark ActorTemplates; # raise it for suites that exercise glutton's WriteRAM path. # actorCPU Optional (default "1"). CPU limit on the benchmark # ActorTemplates. The scheduler books it against each @@ -43,8 +43,8 @@ # or limits). Memory request and limit on each WorkerPool # pod. Use it to fix how many workers share a node: one per # node needs more than half the node's allocatable memory. -# The limit is also the memory the worker reports as its -# actor capacity. +# The limit, less what the worker keeps for itself, is +# the memory it reports as its actor capacity. # workerWaitTimeout # Optional (default 300, set by workloads/deploy.sh). # Whole seconds to wait after the WorkerPool deploy for @@ -52,14 +52,14 @@ # large workerCount fleets whose image pulls or scheduling # outrun the default. # ateArgs Optional. List of strings appended verbatim to -# `hack/install-ate.sh deploy ate-system` when this +# `ate-setup deploy ate-system` when this # test's substrate is deployed (e.g. ["--atenet-dataplane=agentgateway"]). # The deploy requires --credential-provider; set it here # as one YAML string, '--credential-provider={"name":"k8s.io"}' # (no shell quoting: the args are passed without a shell), # or as ATE_CREDENTIAL_PROVIDER in .ate-dev-env.sh. # isolate Optional. Isolate mode: installs substrate with the fake -# data plane (hack/install-ate.sh deploy ate-system --fake-data-plane), which +# data plane (ate-setup deploy ate-system --fake-data-plane), which # serves the WorkerPool without worker pods: fake-atelet # takes atelet's place on every atelet node and answers each # lifecycle call after a delay, and fake-workersync honors @@ -128,8 +128,9 @@ # args Optional. Postgres settings as name=value (e.g. # commit_delay=100), applied in the restart stats makes. # The block leaves placement to the install. A capacity run sets -# --cordon-control-plane in ateArgs, which keeps ate-api-server and Postgres -# off the nodes that run the load, on the tainted pools that flag expects. +# --cordon-system-infra-components in ateArgs, which keeps ate-api-server and +# Postgres off the nodes that run the load, on the tainted pools that flag +# expects. # # ---- type: nighthawk-ingress -------------------------------------------------------- # @@ -174,9 +175,9 @@ tests: # GluttonUser fills the actor's RAM to --mem-target through the # glutton WriteRAM API before the first suspend, so every suspend/resume # cycle is measured against a realistically-sized actor. actorMemory = - # target + headroom (glutton itself, the guest, and on microvm the 128Mi - # VMM reserve). The golden snapshot and cold boot stay small: the working - # set exists from the first fill onward. --mem-churn re-randomizes 64Mi + # target + headroom (glutton itself and the guest). The golden snapshot + # and cold boot stay small: the working set exists from the first fill + # onward. --mem-churn re-randomizes 64Mi # each cycle through an advancing window (WriteRAM rotate), so repeated # suspends snapshot changing memory rather than a static set and the dirty # window moves across the array. --mem-read walks the whole working set @@ -722,7 +723,7 @@ tests: isolate: delay: 150ms ateArgs: - - "--cordon-control-plane" + - "--cordon-system-infra-components" controlPlane: apiServer: replicas: 1 @@ -750,7 +751,7 @@ tests: isolate: delay: 150ms ateArgs: - - "--cordon-control-plane" + - "--cordon-system-infra-components" controlPlane: apiServer: replicas: 1 diff --git a/benchmarking/automation/util.py b/benchmarking/automation/util.py index ad6a183906..330d21b4f5 100644 --- a/benchmarking/automation/util.py +++ b/benchmarking/automation/util.py @@ -114,7 +114,7 @@ def isolate_block(test: dict) -> dict | None: def isolate_install_args(test: dict) -> list[str]: - """Return the hack/install-ate.sh flags a tests.yaml entry's isolate block + """Return the ate-setup flags a tests.yaml entry's isolate block installs the fake data plane with, or none without a block. The installer only chooses the data plane; the block's tuning goes to deploy.sh.""" return [] if isolate_block(test) is None else ["--fake-data-plane"] diff --git a/benchmarking/deploy_locust.sh b/benchmarking/deploy_locust.sh index fb4d335bdb..d9421a6b7c 100755 --- a/benchmarking/deploy_locust.sh +++ b/benchmarking/deploy_locust.sh @@ -29,7 +29,7 @@ WORKER_COUNT=1 SANDBOX_CLASS=gvisor SKIP_BUILD=0 OTLP_ENDPOINT="" -# Empty keeps the default in workloads/deploy.sh (256Mi, the microvm minimum). +# Empty keeps the default in workloads/deploy.sh (256Mi). ACTOR_MEMORY="" # Empty keeps the default in workloads/deploy.sh (1 CPU). ACTOR_CPU="" @@ -50,7 +50,7 @@ usage() { echo " --otlp-endpoint URL Forwarded to workloads/deploy.sh. The address to which an" echo " instrumented actor container sends telemetry." echo " --actor-memory SIZE Forwarded to workloads/deploy.sh. Memory limit for the" - echo " benchmark ActorTemplates (default: 256Mi, the microvm minimum)." + echo " benchmark ActorTemplates (default: 256Mi)." echo " --actor-cpu CPU Forwarded to workloads/deploy.sh. CPU limit for the benchmark" echo " ActorTemplates (default: 1)." echo " --worker-memory SIZE Forwarded to workloads/deploy.sh. Memory request and limit" @@ -63,7 +63,7 @@ usage() { echo " -h|--help Show this help message" echo "" echo "Environment:" - echo " PROJECT_ID, BUCKET_NAME, etc. are read from .ate-dev-env.sh by the" + echo " PROJECT_ID etc. are read from .ate-dev-env.sh by the" echo " scripts this wrapper invokes." } diff --git a/benchmarking/nighthawk-egress/README.md b/benchmarking/nighthawk-egress/README.md index df62df9284..3377329c26 100644 --- a/benchmarking/nighthawk-egress/README.md +++ b/benchmarking/nighthawk-egress/README.md @@ -36,19 +36,19 @@ variable when deploying ATE. 1. **Cluster with Agent Substrate deployed**: - **Existing cluster** (with `.ate-dev-env.sh` configured): ```bash - hack/install-ate.sh deploy ate-system \ + go run ./cmd/ate-setup deploy ate-system \ --credential-provider='{"name":"k8s.io"}' \ --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/nighthawk/sa/default ``` - **Local `kind` cluster**: ```bash - hack/install-ate-kind.sh deploy ate-system \ + go run ./cmd/ate-setup --kind deploy ate-system \ --credential-provider='{"enabled":false}' \ --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/nighthawk/sa/default ``` - To set the `atenet-egress` Envoy worker thread count (`--concurrency`), set `E2E_ATE_ENVOY_CONCURRENCY` when deploying: ```bash - E2E_ATE_ENVOY_CONCURRENCY=1 hack/install-ate-kind.sh deploy ate-system --credential-provider='{"name":"k8s.io"}' --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/nighthawk/sa/default + E2E_ATE_ENVOY_CONCURRENCY=1 go run ./cmd/ate-setup --kind deploy ate-system --credential-provider='{"name":"k8s.io"}' --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/nighthawk/sa/default ``` - Provision CA bundle which includes cluster's trust bundle, so that egress gateway can trust nighhawk's test server cert, run: ```bash @@ -79,7 +79,7 @@ variable when deploying ATE. In a separate terminal, forward local port `19980` to the `traffic-driver-invoker` service: ```bash -kubectl -n ate-system port-forward svc/traffic-driver-invoker 19980:19980 +kubectl -n nighthawk port-forward svc/traffic-driver-invoker 19980:19980 ``` ## Running the Benchmark @@ -87,7 +87,7 @@ kubectl -n ate-system port-forward svc/traffic-driver-invoker 19980:19980 Run `nighthawk_client` against the port-forwarded `traffic-driver-invoker` endpoint (`grpc://127.0.0.1:19980`), which routes the gRPC request to `nighthawk_service` inside the `nighthawk/tester` actor: ```bash -docker run --rm --network=host --entrypoint /usr/local/bin/nighthawk_client envoyproxy/nighthawk-dev:latest --nighthawk-service grpc://127.0.0.1:19980 --rps 1000 --concurrency 1 --protocol http1 --duration 60 http://nighthawk-server.ate-system.svc:80 +docker run --rm --network=host --entrypoint /usr/local/bin/nighthawk_client envoyproxy/nighthawk-dev:latest --nighthawk-service grpc://127.0.0.1:19980 --rps 1000 --concurrency 1 --protocol http1 --duration 60 http://nighthawk-server.nighthawk.svc:80 ``` See [Nighthawk CLI help](https://github.com/envoyproxy/nighthawk#using-the-nighthawk-client-cli) for all command line options. @@ -96,9 +96,9 @@ See [Nighthawk CLI help](https://github.com/envoyproxy/nighthawk#using-the-night | Egress Mode | Target URL | Port | |---|---|---| -| Plaintext HTTP | `http://nighthawk-server.ate-system.svc:80` | `80` | -| HTTPS (TLS MITM) | `https://nighthawk-server.ate-system.svc:443` | `443` | -| TLS Passthrough | `https://nighthawk-server.ate-system.svc:8443` | `8443` | +| Plaintext HTTP | `http://nighthawk-server.nighthawk.svc:80` | `80` | +| HTTPS (TLS MITM) | `https://nighthawk-server.nighthawk.svc:443` | `443` | +| TLS Passthrough | `https://nighthawk-server.nighthawk.svc:8443` | `8443` | ## Cleanup @@ -108,8 +108,10 @@ Delete the `nighthawk/tester` actor, actor template, atespace, and worker pool: ./benchmarking/nighthawk-egress/deploy-traffic-driver.sh --delete ``` -Delete `nighthawk-server` and `traffic-driver-invoker` from `ate-system`: +Delete `nighthawk-server` and `traffic-driver-invoker` from `nighthawk`: ```bash ./benchmarking/nighthawk-egress/deploy.sh --delete ``` + +Neither script deletes the `nighthawk` namespace. Delete it with `kubectl delete namespace nighthawk` after `deploy-traffic-driver.sh --delete`, because the actor, template, and atespace live in the ate API, not in the namespace. diff --git a/benchmarking/nighthawk-egress/deploy-traffic-driver.sh b/benchmarking/nighthawk-egress/deploy-traffic-driver.sh index af5698742f..19d3aeb9ba 100755 --- a/benchmarking/nighthawk-egress/deploy-traffic-driver.sh +++ b/benchmarking/nighthawk-egress/deploy-traffic-driver.sh @@ -29,12 +29,12 @@ ATESPACE="nighthawk" TEMPLATE_NAME="traffic-driver" ACTOR_NAME="tester" POOL_MANIFEST="${SCRIPT_DIR}/traffic-driver.yaml.tmpl" +NAMESPACE_MANIFEST="${SCRIPT_DIR}/namespace.yaml" MANIFEST="${SCRIPT_DIR}/traffic-driver-template.yaml.tmpl" EGRESS_POLICY_MANIFEST="${SCRIPT_DIR}/egress-policy.yaml" KUBECTL_ATE_BIN="kubectl-ate" WAIT_TIMEOUT_SECS=300 CONTEXT="${KUBECTL_CONTEXT:-}" -BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" KO_DOCKER_REPO="${KO_DOCKER_REPO:-localhost:5001}" usage() { @@ -43,7 +43,6 @@ usage() { echo "Options:" echo " --deploy Deploy the ${ATESPACE}/${TEMPLATE_NAME} worker pool, actor template, and ${ACTOR_NAME} actor (default)" echo " --delete Delete the ${ACTOR_NAME} actor, template, atespace, and worker pool" - echo " --bucket-name BUCKET GCS bucket name for actor snapshots (default: \$BUCKET_NAME)" echo " --ko-docker-repo REPO Container registry for ko image builds (default: \$KO_DOCKER_REPO)" echo " --context CONTEXT kubectl context to use (default: \$KUBECTL_CONTEXT or current context)" echo " --wait-timeout SECONDS Timeout waiting for worker pool and golden snapshot readiness (default: ${WAIT_TIMEOUT_SECS})" @@ -59,13 +58,6 @@ while [[ "$#" -gt 0 ]]; do --delete) action="delete" ;; - --bucket-name) - shift - BUCKET_NAME="$1" - ;; - --bucket-name=*) - BUCKET_NAME="${1#*=}" - ;; --ko-docker-repo) shift KO_DOCKER_REPO="$1" @@ -149,16 +141,15 @@ wait_template_ready() { } deploy() { - if [[ -z "${BUCKET_NAME}" ]]; then - echo "Error: BUCKET_NAME is not set. Pass --bucket-name or configure .ate-dev-env.sh." >&2 - exit 1 - fi if [[ -z "${KO_DOCKER_REPO}" ]]; then echo "Error: KO_DOCKER_REPO is not set. Pass --ko-docker-repo or configure .ate-dev-env.sh." >&2 exit 1 fi export KO_DOCKER_REPO + echo "Deploying namespace from ${NAMESPACE_MANIFEST}..." + run_kubectl apply -f "${NAMESPACE_MANIFEST}" + echo "Deploying worker pool from ${POOL_MANIFEST}..." hack/run-tool.sh ko resolve -f "${POOL_MANIFEST}" | run_kubectl apply -f - echo "Waiting for worker pool ${ATESPACE}/${TEMPLATE_NAME} to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." @@ -177,8 +168,7 @@ deploy() { run_kubectl_ate delete actor-template "${TEMPLATE_NAME}" -a "${ATESPACE}" >/dev/null 2>&1 || true echo "Creating actor template ${ATESPACE}/${TEMPLATE_NAME} from ${MANIFEST}..." - sed "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" "${MANIFEST}" \ - | run_kubectl_ate create actor-template -f - + run_kubectl_ate create actor-template -f "${MANIFEST}" wait_template_ready diff --git a/benchmarking/nighthawk-egress/deploy.sh b/benchmarking/nighthawk-egress/deploy.sh index f10f2d2441..d0a0f086b4 100755 --- a/benchmarking/nighthawk-egress/deploy.sh +++ b/benchmarking/nighthawk-egress/deploy.sh @@ -17,6 +17,7 @@ set -o errexit -o nounset -o pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +NAMESPACE_MANIFEST="${SCRIPT_DIR}/namespace.yaml" SERVER_MANIFEST="${SCRIPT_DIR}/nighthawk-server.yaml" INVOKER_MANIFEST="${SCRIPT_DIR}/traffic-driver-invoker.yaml" WAIT_TIMEOUT_SECS=120 @@ -26,8 +27,8 @@ usage() { echo "Usage: $0 [options]" echo "" echo "Options:" - echo " --deploy Deploy nighthawk-server and traffic-driver-invoker in ate-system (default)" - echo " --delete Delete nighthawk-server and traffic-driver-invoker from ate-system" + echo " --deploy Deploy nighthawk-server and traffic-driver-invoker in nighthawk (default)" + echo " --delete Delete nighthawk-server and traffic-driver-invoker from nighthawk" echo " --context CONTEXT kubectl context to use (default: \$KUBECTL_CONTEXT or current context)" echo " --wait-timeout SECONDS Rollout wait timeout in seconds (default: ${WAIT_TIMEOUT_SECS})" echo " -h, --help Show this help message" @@ -80,13 +81,15 @@ if [[ -n "${CONTEXT}" ]]; then fi if [[ "${action}" == "deploy" ]]; then + echo "Deploying namespace from ${NAMESPACE_MANIFEST}..." + "${kubectl_cmd[@]}" apply -f "${NAMESPACE_MANIFEST}" echo "Deploying Nighthawk server from ${SERVER_MANIFEST}..." "${kubectl_cmd[@]}" apply -f "${SERVER_MANIFEST}" echo "Deploying traffic-driver-invoker from ${INVOKER_MANIFEST}..." "${kubectl_cmd[@]}" apply -f "${INVOKER_MANIFEST}" - echo "Waiting for deployment/nighthawk-server in ate-system to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." + echo "Waiting for deployment/nighthawk-server in nighthawk to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." "${kubectl_cmd[@]}" -n nighthawk rollout status deployment/nighthawk-server --timeout="${WAIT_TIMEOUT_SECS}s" - echo "Waiting for deployment/traffic-driver-invoker in ate-system to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." + echo "Waiting for deployment/traffic-driver-invoker in nighthawk to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." "${kubectl_cmd[@]}" -n nighthawk rollout status deployment/traffic-driver-invoker --timeout="${WAIT_TIMEOUT_SECS}s" elif [[ "${action}" == "delete" ]]; then echo "Deleting Nighthawk server and traffic-driver-invoker..." diff --git a/internal/plugins/gcp-secret-manager/config/service.yaml b/benchmarking/nighthawk-egress/namespace.yaml similarity index 64% rename from internal/plugins/gcp-secret-manager/config/service.yaml rename to benchmarking/nighthawk-egress/namespace.yaml index 743acd4a21..22f70c340f 100644 --- a/internal/plugins/gcp-secret-manager/config/service.yaml +++ b/benchmarking/nighthawk-egress/namespace.yaml @@ -12,19 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Required for the pod to start: substrate issues the serving certificate only -# for pods a Service selects. apiVersion: v1 -kind: Service +kind: Namespace metadata: - name: gsm-credential-provider - namespace: ate-system -spec: - type: ClusterIP - selector: - app: gsm-credential-provider - ports: - - name: grpc - port: 50051 - targetPort: grpc - protocol: TCP + name: nighthawk diff --git a/benchmarking/nighthawk-egress/nighthawk-server.yaml b/benchmarking/nighthawk-egress/nighthawk-server.yaml index 1f1cc7f3ec..ee5a2ed061 100644 --- a/benchmarking/nighthawk-egress/nighthawk-server.yaml +++ b/benchmarking/nighthawk-egress/nighthawk-server.yaml @@ -15,7 +15,7 @@ # In-cluster Envoy Nighthawk test server used as the origin target for egress # benchmarks. Exposes plaintext HTTP on :80 and :8080 and TLS-terminated HTTPS # on :443 and :8443 using a serving certificate issued by the cluster's -# servicedns pod-certificate signer (SAN: nighthawk-server.ate-system.svc). +# servicedns pod-certificate signer (SAN: nighthawk-server.nighthawk.svc). apiVersion: v1 kind: ConfigMap metadata: @@ -297,8 +297,8 @@ spec: podcert.ate.dev/canarying: live path: trust-bundle.pem --- -# Selecting Service in ate-system required by servicedns.podcert.ate.dev/identity -# to mint a serving certificate with SAN nighthawk-server.ate-system.svc. +# Selecting Service in nighthawk required by servicedns.podcert.ate.dev/identity +# to mint a serving certificate with SAN nighthawk-server.nighthawk.svc. apiVersion: v1 kind: Service metadata: diff --git a/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl b/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl index 27c68f984b..0960539262 100644 --- a/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl +++ b/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl @@ -14,7 +14,7 @@ # Substrate ActorTemplate resource (protojson-shaped ateapi.ActorTemplate, # NOT the ate.dev/v1 CRD) for the Nighthawk egress traffic-driver actor. -# Applied with `kubectl-ate create actor-template -f -` by +# Applied with `kubectl-ate create actor-template -f` by # deploy-traffic-driver.sh; the `nighthawk` atespace must exist. # # Runs Envoy's `nighthawk_service` gRPC server on port 80 (the default actor @@ -65,4 +65,3 @@ resources: memoryBytes: 1073741824 # 1Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/nighthawk/traffic-driver/ diff --git a/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl b/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl index 6853b65fbb..7d622b79f1 100644 --- a/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl +++ b/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl @@ -17,13 +17,6 @@ # traffic-driver-template.yaml.tmpl and is created through the ate API with # `kubectl-ate create actor-template`. -apiVersion: v1 -kind: Namespace -metadata: - name: nighthawk - ---- - apiVersion: ate.dev/v1 kind: WorkerPool metadata: diff --git a/benchmarking/nighthawk-ingress/README.md b/benchmarking/nighthawk-ingress/README.md index 1dd1d9bb14..21b1c75f70 100644 --- a/benchmarking/nighthawk-ingress/README.md +++ b/benchmarking/nighthawk-ingress/README.md @@ -120,7 +120,7 @@ requests 2×`envoyCpu` CPUs; the runner requests `clientConcurrency`+1). ```bash # .ate-dev-env.sh at the repo root, then: -hack/install-ate.sh deploy ate-system \ +go run ./cmd/ate-setup deploy ate-system \ --credential-provider='{"name":"k8s.io"}' \ --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner benchmarking/workloads/deploy.sh --deploy --worker-count 50 --sandbox-class gvisor @@ -149,7 +149,7 @@ Flags: `--envoy-cpu`, `--actors`, `--tail-latency-slo-ms`, `--atespace`, Notes: - The script does not redeploy substrate — if substrate code changed, - rerun `hack/install-ate.sh deploy ate-system` first. + rerun `go run ./cmd/ate-setup deploy ate-system` first. - `--atespace` isolates actor *fleets* between experiments sharing a cluster; capacity runs themselves still need the router exclusively (the cpu pin is global and the run saturates it). diff --git a/benchmarking/nighthawk-ingress/run-dev.sh b/benchmarking/nighthawk-ingress/run-dev.sh index e20dd51e1a..7990c874ca 100755 --- a/benchmarking/nighthawk-ingress/run-dev.sh +++ b/benchmarking/nighthawk-ingress/run-dev.sh @@ -80,7 +80,7 @@ docker info >/dev/null 2>&1 || { kubectl get deployment atenet-router -n ate-system >/dev/null 2>&1 || { echo "ERROR: substrate is not deployed on ${CLUSTER_NAME}. Run:" >&2 - echo " hack/install-ate.sh deploy ate-system --credential-provider='{\"name\":\"k8s.io\"}' --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner" >&2 + echo " go run ./cmd/ate-setup deploy ate-system --credential-provider='{\"name\":\"k8s.io\"}' --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner" >&2 exit 1 } diff --git a/benchmarking/telemetry/README.md b/benchmarking/telemetry/README.md index a5d991b037..e097b3e8b0 100644 --- a/benchmarking/telemetry/README.md +++ b/benchmarking/telemetry/README.md @@ -73,7 +73,7 @@ holds the same count connector and gives the same two counts. A meter in front of it would add a hop and measure nothing new. The meter exists for GKE only, where the managed collector cannot hold a connector. -`hack/install-ate.sh` installs the collector and a Prometheus with it, thus a +`ate-setup --kind` installs the collector and a Prometheus with it, thus a kind cluster needs no other step: ```bash diff --git a/benchmarking/workloads/deploy.sh b/benchmarking/workloads/deploy.sh index c18328dfca..b934326ead 100755 --- a/benchmarking/workloads/deploy.sh +++ b/benchmarking/workloads/deploy.sh @@ -23,12 +23,6 @@ if [[ -f .ate-dev-env.sh ]]; then source .ate-dev-env.sh fi -# Ensure BUCKET_NAME is set -if [[ -z "${BUCKET_NAME:-}" ]]; then - echo "Error: BUCKET_NAME environment variable is not set." >&2 - exit 1 -fi - MANIFEST_DIR="benchmarking/workloads/manifests" POOL_MANIFEST="${MANIFEST_DIR}/workloads.yaml.tmpl" # Generated sweperf ActorTemplates; see manifests/sweperf/generate_sweperf_templates.py. @@ -39,8 +33,8 @@ SWEPERF_TEMPLATE_DIR="${MANIFEST_DIR}/sweperf/generated" # overrides the default set — the usermem and kernelmem templates (for the # matching locust tests) are not deployed by default. The glutton-storage # template is added when a StorageClass is set, because it needs a CSI driver -# that not every cluster has. -read -r -a TEMPLATES <<<"${WORKLOAD_TEMPLATES:-sleep glutton glutton-durdir-data glutton-durdir-full ate-env}" +# that not every cluster has; ate-env is added when ATE_ENV_IMAGE is set. +read -r -a TEMPLATES <<<"${WORKLOAD_TEMPLATES:-sleep glutton glutton-durdir-data glutton-durdir-full}" # template_manifest prints the manifest path for ActorTemplate NAME. template_manifest() { @@ -57,25 +51,24 @@ fi WORKER_COUNT=1 SANDBOX_CLASS="gvisor" -# Actor memory limit (ActorTemplate resources.limits.memory). The default -# is the smallest size microvm admits (128Mi VMM reserve + 128Mi guest floor), -# since a bigger guest drags more page cache into every memory snapshot. Raise -# it for RAM-consuming suites. +# Actor memory limit (ActorTemplate resources.limits.memory). Small by +# default, since a bigger guest drags more page cache into every memory +# snapshot. Raise it for RAM-consuming suites. ACTOR_MEMORY="256Mi" # Actor CPU limit (ActorTemplate resources.limits.cpu). The scheduler books # it against the worker's CPU capacity, so it bounds how many actors a worker # hosts at once as much as the memory limit does. ACTOR_CPU="1" # Worker pod memory (WorkerPool spec.template.resources), set as both request -# and limit. Empty leaves the pod unsized. The limit is also the memory the -# worker reports as its actor capacity. +# and limit. Empty leaves the pod unsized. The limit, less what the worker +# keeps for itself, is the memory it reports as its actor capacity. WORKER_MEMORY="" # The StorageClass for the glutton-storage external volume. Empty means the # template is not deployed. STORAGE_CLASS_NAME="${STORAGE_CLASS_NAME:-}" -# The image for the ate-env benchmark ActorTemplate. Defaults to the pinned -# release guest image; set ATE_ENV_IMAGE to override with a custom build. -ATE_ENV_IMAGE="${ATE_ENV_IMAGE:-us-docker.pkg.dev/gke-substrate-release/benchmark/ate-env-guest@sha256:1d38934ef1dac71fcb1f1286de734e06ab2e19747279bccbd37bfe69e0d5bc72}" +# The image for the ate-env benchmark ActorTemplate. Empty means the +# template is not deployed. +ATE_ENV_IMAGE="${ATE_ENV_IMAGE:-}" # The address to which an instrumented actor container sends its telemetry. # --otlp-endpoint sets it. Without the flag, resolve_otlp_endpoint reads the # address that the control plane uses. @@ -96,8 +89,7 @@ usage() { echo " --worker-count N Number of WorkerPool replicas (default: 1)" echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)." echo " microvm requires hack/install-microvm-deps.sh --install to have run." - echo " --actor-memory SIZE Memory limit for the benchmark ActorTemplates (default: 256Mi," - echo " the smallest size microvm admits)" + echo " --actor-memory SIZE Memory limit for the benchmark ActorTemplates (default: 256Mi)" echo " --actor-cpu CPU CPU limit for the benchmark ActorTemplates (default: 1)" echo " --worker-memory SIZE Memory request and limit for each WorkerPool pod" echo " (default: unset, the pod is unsized)" @@ -194,8 +186,7 @@ substitute() { if [[ -n "${WORKER_MEMORY}" ]]; then worker_resources="resources: {requests: {memory: \"${WORKER_MEMORY}\"}, limits: {memory: \"${WORKER_MEMORY}\"}}" fi - sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \ - -e "s|\${WORKER_COUNT}|${WORKER_COUNT}|g" \ + sed -e "s|\${WORKER_COUNT}|${WORKER_COUNT}|g" \ -e "s|\${SANDBOX_CLASS}|${SANDBOX_CLASS}|g" \ -e "s|\${OTLP_ENDPOINT}|${OTLP_ENDPOINT}|g" \ -e "s|\${ACTOR_MEMORY}|${ACTOR_MEMORY}|g" \ @@ -477,6 +468,13 @@ if [[ -z "${WORKLOAD_TEMPLATES:-}" ]] \ TEMPLATES+=(glutton-storage) fi +# ate-env joins the default set only when ATE_ENV_IMAGE is set. Delete always +# includes it, because deleting a template that does not exist is a no-op. +if [[ -z "${WORKLOAD_TEMPLATES:-}" ]] \ + && { [[ -n "${ATE_ENV_IMAGE}" ]] || [[ "${action}" == "delete" ]]; }; then + TEMPLATES+=(ate-env) +fi + if [[ "${action}" == "deploy" && -z "${STORAGE_CLASS_NAME}" && " ${TEMPLATES[*]} " == *" glutton-storage "* ]]; then echo "Error: the glutton-storage template needs --storage-class-name" >&2 exit 1 diff --git a/benchmarking/workloads/manifests/ate-env-template.yaml.tmpl b/benchmarking/workloads/manifests/ate-env-template.yaml.tmpl index 2640d0846e..1df6a48b65 100644 --- a/benchmarking/workloads/manifests/ate-env-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/ate-env-template.yaml.tmpl @@ -29,9 +29,7 @@ containers: # # KO_DOCKER_REPO="gcr.io//ate-env-guest" ko build --bare --platform=linux/amd64 ./cmd/ate-env-guest # - # Substituted by benchmarking/workloads/deploy.sh (which defaults ATE_ENV_IMAGE - # to us-docker.pkg.dev/gke-substrate-release/benchmark/ate-env-guest@sha256:1d38934ef1dac71fcb1f1286de734e06ab2e19747279bccbd37bfe69e0d5bc72; - # set ATE_ENV_IMAGE to override with a custom digest). + # Substituted from ATE_ENV_IMAGE by benchmarking/workloads/deploy.sh. image: "${ATE_ENV_IMAGE}" command: - "/ko-app/ate-env-guest" @@ -46,4 +44,3 @@ resources: memoryBytes: "${ACTOR_MEMORY_BYTES}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/ate-env/ diff --git a/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl index d507e18483..67f510453b 100644 --- a/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl @@ -50,4 +50,3 @@ resources: memoryBytes: "${ACTOR_MEMORY_BYTES}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-data/ diff --git a/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl index 0cbc965059..f961cbdbe2 100644 --- a/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl @@ -47,4 +47,3 @@ resources: memoryBytes: "${ACTOR_MEMORY_BYTES}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-full/ diff --git a/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl index ad7dcb28bb..77fc654a40 100644 --- a/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl @@ -57,4 +57,3 @@ resources: memoryBytes: "${ACTOR_MEMORY_BYTES}" snapshotConfig: onCommit: SNAPSHOT_CONTENT_SCOPE_FULL - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-storage/ diff --git a/benchmarking/workloads/manifests/glutton-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-template.yaml.tmpl index 7ed6d25d1b..bdec96ac0f 100644 --- a/benchmarking/workloads/manifests/glutton-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-template.yaml.tmpl @@ -59,4 +59,3 @@ resources: memoryBytes: "${ACTOR_MEMORY_BYTES}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton/ diff --git a/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl b/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl index 7019662db9..df3f93c811 100644 --- a/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl @@ -37,4 +37,3 @@ resources: memoryBytes: 8589934592 # 8Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/kernelmem/ diff --git a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl index 5f9ced3242..19c462199e 100644 --- a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl @@ -26,12 +26,10 @@ containers: command: ["/bin/sleep", "1000000"] # Keep the memory small: anything the micro-VM guest kernel caches rides # along in the memory snapshot, so a bigger guest means slower -# suspend/resume. microvm requires at least 256Mi (128Mi VMM reserve + -# 128Mi guest floor). +# suspend/resume. microvm requires at least 128Mi. resources: limits: cpuMillicores: "${ACTOR_CPU_MILLICORES}" memoryBytes: "${ACTOR_MEMORY_BYTES}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sleep/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl index 33bb0d7738..c641f74f27 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl @@ -14,7 +14,7 @@ # Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py # from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this -# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +# file. metadata: atespace: benchmark-workloads name: sweperf-astropy-7336 @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-astropy-7336/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl index af94623150..4c5a62be26 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl @@ -14,7 +14,7 @@ # Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py # from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this -# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +# file. metadata: atespace: benchmark-workloads name: sweperf-django-11099 @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-django-11099/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl index 040e0486ff..8751fbde91 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl @@ -14,7 +14,7 @@ # Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py # from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this -# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +# file. metadata: atespace: benchmark-workloads name: sweperf-matplotlib-20826 @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-matplotlib-20826/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl index 1a8264c447..98c2222fd8 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl @@ -14,7 +14,7 @@ # Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py # from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this -# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +# file. metadata: atespace: benchmark-workloads name: sweperf-sphinx-8638 @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-sphinx-8638/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl index e869b8c2ee..419de76702 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl @@ -14,7 +14,7 @@ # Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py # from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this -# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +# file. metadata: atespace: benchmark-workloads name: sweperf-sympy-13647 @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-sympy-13647/ diff --git a/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 b/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 index 9fc60b1653..a624026dae 100644 --- a/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 +++ b/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 @@ -14,7 +14,7 @@ # Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py # from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this -# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +# file. metadata: atespace: benchmark-workloads name: {{ name }} @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/{{ name }}/ diff --git a/benchmarking/workloads/manifests/sweperf/test_generate_sweperf_templates.py b/benchmarking/workloads/manifests/sweperf/test_generate_sweperf_templates.py index 2d67908e01..e5c8242c67 100644 --- a/benchmarking/workloads/manifests/sweperf/test_generate_sweperf_templates.py +++ b/benchmarking/workloads/manifests/sweperf/test_generate_sweperf_templates.py @@ -40,8 +40,6 @@ def test_render_fills_task_values(self): self.assertIn("name: sweperf-django-11099", content) self.assertIn(f'image: "{PINNED}"', content) self.assertIn('value: "15"', content) - # deploy.sh fills these, so they must survive rendering. - self.assertIn("${BUCKET_NAME}", content) def test_check_reports_missing_stale_and_orphaned_files(self): rendered = gen.render([{"name": n, "image": PINNED, "steps": 1} diff --git a/benchmarking/workloads/manifests/usermem-template.yaml.tmpl b/benchmarking/workloads/manifests/usermem-template.yaml.tmpl index 5b0bd1fc54..84637bdcb4 100644 --- a/benchmarking/workloads/manifests/usermem-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/usermem-template.yaml.tmpl @@ -35,4 +35,3 @@ resources: memoryBytes: 1610612736 # 1536Mi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/usermem/ diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 5b955ccb98..dfc2c73898 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -140,11 +140,11 @@ snapshot bucket, and let the API server's identity impersonate it. See [the GCP setup guide](../../tools/setup-gcp/README.md#what-create-iam-actually-grants) for the required IAM grants. S3 and RustFS do not need this setting. -The default gVisor config now uses release `20261019.0` (`v5`) for separate +The default gVisor config now uses release `20261026.0` (`v6`) for separate container-rootfs and durable-directory checkpoint bundles. Upgrade the sandbox config together with ateworker-gvisor; custom configs must pin a release with the same checkpoint support and supply the `gvisor` release tarball asset. Bare `runsc` assets are no longer accepted. -This replaces the previous `v4` config. Existing `v4` memory snapshots need a +This replaces the previous `v5` config. Existing `v5` memory snapshots need a compatible runtime; falling back to a durable filesystem snapshot does not restore the actor's memory. @@ -154,6 +154,11 @@ initializer Job before running `helm upgrade`. The default Job name is image, and Kubernetes cannot patch an existing Job's pod template. Helm recreates the Job, which keeps the existing bucket and its contents. +The router now requires mTLS on ports 8443 and 8444. Set +`atenetRouter.ingressAllowedSPIFFEIDs` to the client identities allowed to reach +actors; its empty default denies all ingress. Plaintext ports 8080 and 8081 +no longer accept connections. + The chart now denies ingress to the control-plane namespace by default and allows the component, router, egress, storage, and monitoring traffic it needs. Label namespaces containing API clients with `ate.dev/api-access=true`; worker @@ -215,7 +220,7 @@ shared `ate-client` ServiceAccount is a bootstrap global owner so demos and CLI access work. For shared installs, set `ateApi.authorization.bootstrapOwners` to your operators' own principals and grant tenant access with `kubectl ate create access-policy`; anyone who can mint the shared client's token otherwise has -global owner access. See [authorization](../../docs/authorization.md). +global owner access. See [authorization](../../docs/guides/authorization.md). Upstream renumbered protobuf fields, so older clients and binaries are not wire compatible. Upgrade clients and components together: upstream now uses the diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index 9207c69ac9..d5a846760d 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -51,6 +51,10 @@ rules: - apiGroups: ["ate.dev"] resources: ["csidriverconfigs"] verbs: ["get", "watch", "list"] +# Sync failures are reported as Warning Events on the worker pod. +- apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch"] # ClusterTrustBundles referenced by SystemInfo trustBundle data sources are # resolved on the node: atelet reads them through an informer and projects # the sanitized PEM into actors (see cmd/atelet/trustbundle.go). diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index d62a62207e..76cb93ad50 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -33,7 +33,7 @@ data: mode: tunnel accessLog: add: - substrate.connect.authority: source.connectHeaders["host"] + substrate.connect.authority: 'destination.address + ":" + string(destination.port)' # Authorize the actor identity at CONNECT-accept, before any tunnel # (HTTP, TLS, or opaque TCP) is terminated. Fails closed when the # control plane is unreachable. @@ -93,7 +93,7 @@ data: routes: - backends: - dynamic: - target: source.connectHeaders["host"] + target: 'destination.address + ":" + string(destination.port)' policies: substrateEgress: host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index 2bda63273f..761386f4b6 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -36,11 +36,16 @@ data: # not retain an idle connection that may belong to the previous actor. backend: poolMaxSize: 0 + # Actor CONNECT authorities resolve through the worker atunnel. + allowDynamicLoopback: true {{- $traces := include "substrate.otel.signalEndpoint" (list "traces" .) }} {{- $logs := include "substrate.otel.signalEndpoint" (list "logs" .) }} -{{- if or $traces $logs }} frontendPolicies: + networkAuthorization: + rules: + - allow: >- + source.spiffeId in {{ .Values.atenetRouter.ingressAllowedSPIFFEIDs | toJson }} {{- if $traces }} tracing: host: $AGENTGATEWAY_OTLP_TRACES_ADDRESS @@ -55,7 +60,6 @@ data: otlp: host: $AGENTGATEWAY_OTLP_LOGS_ADDRESS protocol: grpc -{{- end }} {{- end }} backends: @@ -76,27 +80,24 @@ data: insecureHost: true gateways: - http: - port: 8080 - protocol: HTTP https: port: 8443 protocol: HTTPS tls: cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + root: /run/podidentity.podcert.ate.dev/trust-bundle.pem routes: - name: substrate-actors gateways: - - http - https matches: - path: pathPrefix: / policies: &substrate-ingress - # Inspect every value before forwarding; substrateIngress reads only - # the first actor header. Apply this to HTTP, gRPC, and CONNECT traffic. + # Reject duplicates in both CONNECT and inner request headers, even + # when actor selection uses only one of those header maps. directResponse: conditional: - condition: >- @@ -119,7 +120,6 @@ data: - backend: /dynamic - name: substrate-actors-grpc gateways: - - http - https matches: &grpc-matches - method: POST @@ -136,11 +136,6 @@ data: # Terminate client CONNECT before internal HTTP routing. binds: - - port: 8081 - tunnelProtocol: connect - listeners: - - protocol: HTTP - routes: [] - port: 8444 tunnelProtocol: connect listeners: @@ -148,6 +143,7 @@ data: tls: cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + root: /run/podidentity.podcert.ate.dev/trust-bundle.pem routes: [] - mode: internal listeners: diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml index 03f8cdab7d..dea6df98f0 100644 --- a/charts/substrate/templates/sandboxconfig-gvisor.yaml +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -33,21 +33,22 @@ metadata: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - defaultVersion: v5 + defaultVersion: v6 versions: - # gVisor release 20261019.0, the first tagged release with - # google/gvisor@b79256080dec (save and restore multiple filesystem checkpoint - # bundles). ateworker-gvisor uses it to save container rootfs overlays - # (rootfs/) and durable-dir volumes (fs/) as separate bundles, and to run the - # sandbox with no root container (google/gvisor#15551). - - name: v5 + # gVisor release 20261026.0. ateworker-gvisor saves container rootfs overlays + # (rootfs/) and durable-dir volumes (fs/) as separate checkpoint bundles. That + # needs google/gvisor@b79256080dec, first in tagged release 20261019.0. + # ateworker-gvisor also runs the sandbox with no root container + # (google/gvisor#15551). This release's restore picks each container's init + # process correctly in such a sandbox (google/gvisor@29b97c66d945). + - name: v6 state: Enabled assets: amd64: gvisor: - url: "gs://gvisor/releases/release/20261019.0/x86_64/gvisor.tar.zstd" - sha256: "8637abfb50f81552af25ca08f7172589c09292a7c881efe8d9e09a9f3b6bec7b" + url: "gs://gvisor/releases/release/20261026.0/x86_64/gvisor.tar.zstd" + sha256: "708afc62f2ee82d19930c703e85f9dab4f27a78c0aec40c0f55eb1040107f1a7" arm64: gvisor: - url: "gs://gvisor/releases/release/20261019.0/aarch64/gvisor.tar.zstd" - sha256: "88556c21468a066d872a055425958c83fcbafc79c6477a195b18070c77f856d0" + url: "gs://gvisor/releases/release/20261026.0/aarch64/gvisor.tar.zstd" + sha256: "6257e49955f13e6caceb2c4181c5dc5406c460e805940c944e67ad65553f5b02" diff --git a/charts/substrate/tests/control_plane_test.yaml b/charts/substrate/tests/control_plane_test.yaml index 6929288577..2f9955400f 100644 --- a/charts/substrate/tests/control_plane_test.yaml +++ b/charts/substrate/tests/control_plane_test.yaml @@ -17,6 +17,7 @@ templates: - ate-api-server.yaml - ate-api-config.yaml - ate-controller.yaml +- atenet-router.yaml - role.yaml - atelet.yaml - k8s-credential-provider.yaml @@ -67,6 +68,12 @@ tests: apiGroups: [ate.dev] resources: [sandboxconfigs, workerpools] verbs: [get, list, watch] + - contains: + path: rules + content: + apiGroups: [events.k8s.io] + resources: [events] + verbs: [create, patch] - it: probes atelet health without exposing host ports template: atelet.yaml documentSelector: @@ -324,3 +331,31 @@ tests: apiGroups: [ate.dev] resources: [workerpools, csidriverconfigs] verbs: [get, watch, list] + +- it: requires a trusted and explicitly allowed router client + template: atenet-router.yaml + documentSelector: + path: kind + value: ConfigMap + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: 'source.spiffeId in \[\]' + - matchRegex: + path: data["config.yaml"] + pattern: 'root: /run/podidentity.podcert.ate.dev/trust-bundle.pem' + - notMatchRegex: + path: data["config.yaml"] + pattern: 'port: 808[01]' +- it: allows the configured router client identities + template: atenet-router.yaml + set: + atenetRouter.ingressAllowedSPIFFEIDs: + - spiffe://cluster.local/ns/client/sa/router-client + documentSelector: + path: kind + value: ConfigMap + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: 'source.spiffeId in \["spiffe://cluster.local/ns/client/sa/router-client"\]' diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index ca2d2b4e39..6cd8426931 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -117,7 +117,7 @@ ateApi: logLevel: info authorization: # Empty defaults to the release's shared ate-client principal. Use explicit - # operator principals for shared installs; see docs/authorization.md. + # operator principals for shared installs; see docs/guides/authorization.md. bootstrapOwners: [] # Defaults to the in-cluster IdP URL; set a stable issuer for external consumers. actorJWTIssuer: "" @@ -143,6 +143,8 @@ ateController: memory: 512Mi atenetRouter: + # mTLS client identities allowed to reach actors. Empty denies all ingress. + ingressAllowedSPIFFEIDs: [] agentgateway: resources: requests: @@ -235,5 +237,5 @@ image: images: rustfs: rustfs/rustfs:1.0.3@sha256:9ca9559533b537d10482fe923288b676ad08f92a7069d9a17327fbae3cab688b awsCli: amazon/aws-cli:2.38.5@sha256:c8156a0e74e13929a42d7e60f0d4e14e7d28d980cd21822505d4bff80e5460c1 - agentgateway: ghcr.io/kagent-dev/substrate/agentgateway:c8550278b2f2@sha256:5703311439f92a898603e2abed32edcbaf6ec659dbb4d617568cef6d9c6fc789 + agentgateway: cr.agentgateway.dev/agentgateway:v1.7.0@sha256:60216aafb03d37f94e952ff3d51d2f630a65b10ecaebdc0a864882f92610c45b busybox: busybox:1.36 diff --git a/cmd/ate-setup/README.md b/cmd/ate-setup/README.md index dafb7671ad..540dc84c5f 100644 --- a/cmd/ate-setup/README.md +++ b/cmd/ate-setup/README.md @@ -7,10 +7,6 @@ go run ./cmd/ate-setup [global flags] [flags] go build -o bin/ate-setup ./cmd/ate-setup ``` -`ate-setup` is the installer. `hack/install-ate.sh` is a shim over it, kept so -that existing command lines and CI jobs keep working; it holds no install logic -of its own. - ## Installing pre-built images By default every image is built from the checkout with `ko`, which is what a @@ -21,13 +17,15 @@ tag they share: ``` ate-setup deploy ate-system \ --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=IDS \ --image-repo registry.example.com/substrate \ --image-tag vX.Y.Z ``` The registry has to hold every image the install deploys. The images a release -publishes to `ghcr.io/agent-substrate/substrate` are not enough on their own; -see [`docs/dev/releasing.md`](../../docs/dev/releasing.md#release-images). An +publishes to `ghcr.io/agent-substrate/substrate` leave out the demos and the +benchmarking fakes; see +[`docs/dev/releasing.md`](../../docs/dev/releasing.md#release-images). An install that needs an image the registry lacks fails naming it. Nothing is built and no registry is pushed to; the manifests still come from @@ -35,10 +33,7 @@ the checkout, which should be the tag being installed. Each reference is pinned to the digest its tag names, so the registry has to be readable from here as well as from the cluster. -- [`commands.md`](commands.md) — every command with its `hack/install-ate.sh` - equivalent. +- [`commands.md`](commands.md) — every command, and the common global flags. - [`differences.md`](differences.md) — where the Go port deliberately behaves differently from the shell scripts it replaced, and what was reproduced exactly. -- [`cli-diff.md`](cli-diff.md) — what a `hack/install-ate.sh` user can observe - after the switch. diff --git a/cmd/ate-setup/cli-diff.md b/cmd/ate-setup/cli-diff.md deleted file mode 100644 index 5f538cd2a9..0000000000 --- a/cmd/ate-setup/cli-diff.md +++ /dev/null @@ -1,4 +0,0 @@ -# `hack/install-ate.sh` after the move to `ate-setup` - -`hack/install-ate.sh` no longer installs anything. It merely directly invokes -`go run ./cmd/ate-setup`. New logic should not be added to it. \ No newline at end of file diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index c47ce3da89..62ddeb12f1 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -1,48 +1,44 @@ # ate-setup commands -Every `ate-setup` command alongside the equivalent `hack/install-ate.sh` flag. - -`ate-setup` is the installer. `hack/install-ate.sh` is a shim that translates -the flags below onto these commands, so an existing invocation keeps working; -this table is how to write it directly, and -[`cli-diff.md`](cli-diff.md) covers what the translation does not cover. +Every `ate-setup` command, and the common global flags. +`go run ./cmd/ate-setup --help` lists them all. ``` go run ./cmd/ate-setup [global flags] [flags] ``` -`hack/install-ate.sh` accepts its flags in any order and runs one action per -flag, in command line order. `ate-setup` runs exactly one command per -invocation, so a shell line that passed several `--deploy-*` flags becomes -several `ate-setup` calls — which is what the shim does with it. +`ate-setup` runs exactly one command per invocation. ## Global flags -These configure every command. `hack/install-ate.sh` collects its equivalents in -a pre-scan pass, so they may appear anywhere on its command line. - -| `ate-setup` | `hack/install-ate.sh` | Notes | -|---|---|---| -| `--kind` | `hack/install-ate-kind.sh`, or `ATE_INSTALL_KIND=true` | Kind overlays, the local registry, and host-architecture image builds | -| `--atenet-dataplane envoy\|agentgateway` | `--atenet-dataplane envoy\|agentgateway` | atenet ingress and egress dataplane (default `envoy`) | -| `--rollout-timeout DURATION` | `--rollout-timeout DURATION` | Readiness timeout for workloads (default `60s`). Unlike the shell flag it also governs the podcertificate-controller and CSI waits, which stay at their 120s default until it is passed | -| `--podcert-workers-per-signer N` | `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer | -| `--cluster-size size0\|size10` | `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent | -| `--ateapi-postgres-storage-class NAME` | `ATE_API_POSTGRES_STORAGE_CLASS=NAME` | StorageClass for the bundled PostgreSQL volume instead of the cluster default, such as `dynamic-rwo` on GKE so C4 and N4 nodes can attach it. An existing install must delete the postgres StatefulSet and its claim first | -| `--cordon-control-plane` | `--cordon-control-plane` | Keep the control plane off the worker nodes. Assumes a small shared pool labeled and tainted `ate.dev/workloadType=ate-control-plane:NoSchedule`, across which each workload's replicas are spread, and a one-node pool labeled and tainted `ate.dev/workloadType=ate-postgres:NoSchedule` for postgres alone. `ATE_INSTALL_CORDON_CONTROL_PLANE=true` when the flag is absent | -| `--fake-data-plane` | `--fake-data-plane` | Install the fake data plane for control-plane benchmarks: fake-atelet takes atelet's place in its DaemonSet, taking its objectstore-plugin's storage settings, ate-controller is scaled to zero, and fake-workersync honors every WorkerPool with fake Workers on the fake-atelet nodes. Reaches `deploy ate-system`, `deploy atelet`, and `deploy ate-controller`; only `deploy ate-system` switches a cluster between the real and fake data planes, and the other two refuse a data plane other than the installed one. `deploy ate-system --fake-data-plane` refuses a cluster whose WorkerPools have worker pods. fake-atelet runs with its defaults; a benchmark tunes it through the optional `fake-atelet-tuning` ConfigMap, whose `FAKE_ATELET_*` variables set its flags, and which no install creates or resets. Actors do not exist; never use it on a cluster that serves real actors. `ATE_INSTALL_FAKE_DATA_PLANE=true` when the flag is absent | -| `--ingress-allowed-spiffe-ids IDS` | `--ingress-allowed-spiffe-ids IDS`, or `ATE_INGRESS_ALLOWED_SPIFFE_IDS=IDS` | Required by `deploy ate-system` and `deploy atenet` with `--atenet-dataplane=envoy`. Comma-separated SPIFFE IDs (exact match) of the clients the ingress router lets reach actors over its TLS ports; any listed client may reach any actor. Appended to the router as its `--ingress-allowed-spiffe-ids`. The podidentity signer issues `spiffe://cluster.local/ns//sa/` | -| `--experimental-additional-egress-extproc-service NS/SVC:PORT` | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | -| `--credential-provider JSON` | `--credential-provider JSON`, or `ATE_CREDENTIAL_PROVIDER` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/egress-credential-injection.md`](../../docs/egress-credential-injection.md) | -| `--ateapi-log-level LEVEL` | `--ateapi-log-level LEVEL`, or `ATE_API_LOG_LEVEL=LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). The server picks up a change without restarting | -| `--otlp-endpoint URL` | `--otlp-endpoint URL`, or `ATE_OTLP_ENDPOINT=URL` | Send control plane telemetry to `URL` instead of the cluster default (see [`benchmarking/telemetry/README.md`](../../benchmarking/telemetry/README.md)) | -| `--preview GATES` | `--preview GATES`, or `ATE_PREVIEW=GATES` | Preview gates passed as `--preview` to every substrate component: comma-separated gate names, or `ALL` for all. `ATE_PREVIEW` or `config.preview` when the flag is absent. An explicit `--preview=''` turns them all off, even when `ATE_PREVIEW` is set | -| `--context NAME` | `KUBECTL_CONTEXT=NAME` | Kubeconfig context; still defaults to `KUBECTL_CONTEXT` | -| `--kubeconfig PATH` | `KUBECONFIG=PATH` | Explicit kubeconfig path | -| `--no-dev-env` | `NO_DEV_ENV=1` | Skip `.ate-dev-env.sh` at the repository root | -| `--version` / `-v` | — | New; the shell installer had no version | -| `--image-repo REPO` | — | New. Install pre-built images from `REPO` instead of building them with `ko` | -| `--image-tag TAG` | — | New. The tag those images carry. Each of the two requires the other | +These configure every command, and may appear before or after it, except +`--version`, which takes no command. +[`docs/guides/operator-install.md`](../../docs/guides/operator-install.md#settings) lists +every setting with its environment variable. + +| Flag | Notes | +|---|---| +| `--kind` | Kind overlays, the local registry, and host-architecture image builds. `ATE_INSTALL_KIND=true` when the flag is absent | +| `--atenet-dataplane envoy\|agentgateway` | atenet ingress and egress dataplane (default `envoy`) | +| `--rollout-timeout DURATION` | Readiness timeout for workloads (default `60s`). It also governs the podcertificate-controller and CSI waits, which stay at their 120s default until it is passed | +| `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer | +| `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent | +| `--ateapi-postgres-storage-class NAME` | StorageClass for the bundled PostgreSQL volume instead of the cluster default, such as `dynamic-rwo` on GKE so C4 and N4 nodes can attach it. An existing install must delete the postgres StatefulSet and its claim first. `ATE_API_POSTGRES_STORAGE_CLASS` when the flag is absent | +| `--cordon-system-infra-components` | Keep the system infra components, such as ate-api-server, atenet-router, atenet-egress and postgres, off the worker nodes. Assumes a small shared pool labeled and tainted `ate.dev/workloadType=ate-system-infra:NoSchedule`, across which each workload's replicas are spread, and a one-node pool labeled and tainted `ate.dev/workloadType=ate-postgres:NoSchedule` for postgres alone. `ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS=true` when the flag is absent | +| `--fake-data-plane` | Install the fake data plane for control-plane benchmarks: fake-atelet takes atelet's place in its DaemonSet, taking its objectstore-plugin's storage settings, ate-controller is scaled to zero, and fake-workersync honors every WorkerPool with fake Workers on the fake-atelet nodes. Reaches `deploy ate-system`, `deploy atelet`, and `deploy ate-controller`; only `deploy ate-system` switches a cluster between the real and fake data planes, and the other two refuse a data plane other than the installed one. `deploy ate-system --fake-data-plane` refuses a cluster whose WorkerPools have worker pods. fake-atelet runs with its defaults; a benchmark tunes it through the optional `fake-atelet-tuning` ConfigMap, whose `FAKE_ATELET_*` variables set its flags, and which no install creates or resets. Actors do not exist; never use it on a cluster that serves real actors. `ATE_INSTALL_FAKE_DATA_PLANE=true` when the flag is absent | +| `--ingress-allowed-spiffe-ids IDS` | Comma-separated SPIFFE IDs (exact match) of the clients the ingress router lets reach actors; any listed client may reach any actor. Unset, no client may reach actors through the router. Both dataplanes require a Pod Identity CA client certificate on HTTPS and TLS CONNECT. The SPIFFE URI SAN allowlist may be checked during or after TLS, including after CONNECT acceptance, but unlisted clients must be denied before actor routing. The podidentity signer issues `spiffe://cluster.local/ns//sa/`. `ATE_INGRESS_ALLOWED_SPIFFE_IDS` when the flag is absent | +| `--egress-gateway-extra-spiffe-ids IDS` | Comma-separated SPIFFE IDs that hold ate-api-server's `egress_gateway` role next to atenet-egress, such as a credential provider that mints actor JWTs. ate-setup passes atenet-egress's ID and then each listed ID as repeated `--egress-gateway-spiffe-id` flags; unset passes none, leaving ate-api-server's default. `ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS` when the flag is absent | +| `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | +| `--credential-provider JSON` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/guides/egress-credential-injection.md`](../../docs/guides/egress-credential-injection.md). `ATE_CREDENTIAL_PROVIDER` when the flag is absent | +| `--ateapi-log-level LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). The server picks up a change without restarting. `ATE_API_LOG_LEVEL` when the flag is absent | +| `--otlp-endpoint URL` | Send control plane telemetry to `URL` instead of the cluster default (see [`benchmarking/telemetry/README.md`](../../benchmarking/telemetry/README.md)). `ATE_OTLP_ENDPOINT` when the flag is absent | +| `--preview GATES` | Preview gates passed as `--preview` to every substrate component: comma-separated gate names, or `ALL` for all. `ATE_PREVIEW` or `config.preview` when the flag is absent. An explicit `--preview=''` turns them all off, even when `ATE_PREVIEW` is set | +| `--context NAME` | Kubeconfig context. `KUBECTL_CONTEXT` when the flag is absent | +| `--kubeconfig PATH` | Explicit kubeconfig path. `KUBECONFIG` when the flag is absent | +| `--no-dev-env` | Skip `.ate-dev-env.sh` at the repository root. `NO_DEV_ENV=1` when the flag is absent | +| `--version` / `-v` | Print the version | +| `--image-repo REPO` | Install pre-built images from `REPO` instead of building them with `ko` | +| `--image-tag TAG` | The tag those images carry. Each of the two requires the other | Both have an environment equivalent, read when the flag is absent: `ATE_IMAGE_REPO` and `ATE_IMAGE_TAG`. @@ -56,9 +52,7 @@ replaces an existing cluster with the same name. Deploy the control plane before deploying demos or benchmarks. These commands check for `deployment/ate-controller` in the configured system namespace before creating workload resources. An existing Deployment passes this check; normal -rollout waits still diagnose workloads that cannot become ready. The shell shim -runs actions in argument order, so put `--deploy-ate-system` before demo or -benchmark deployment flags when combining them on one command line. +rollout waits still diagnose workloads that cannot become ready. ## Installing pre-built images @@ -69,13 +63,15 @@ unchanged. ``` ate-setup deploy ate-system \ --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=IDS \ --image-repo REPO \ --image-tag TAG ``` installs images already published to `REPO` instead, and never invokes `ko`. The manifests still come from the checkout, so this needs one; what it removes -is the build, the Go toolchain, and write access to a registry. +is the build and write access to a registry. Releases ship no `ate-setup` +binary, so running it from the checkout still needs Go. `REPO` has to hold every image the install deploys, all under `TAG`, including `envoy-dataplane`, which a build from source builds from a Dockerfile with @@ -83,7 +79,7 @@ is the build, the Go toolchain, and write access to a registry. needs an image `REPO` lacks fails naming it. The images a release publishes to `ghcr.io/agent-substrate/substrate`, with -`publish release-images`, are not a complete install; see +`publish release-images`, leave out the demos and the benchmarking fakes; see [`docs/dev/releasing.md`](../../docs/dev/releasing.md#release-images). When an image releases do not publish is missing from `REPO`, the error says so. @@ -103,17 +99,15 @@ that already names a manifest is used as written, and is not looked up. ## Deploy -| `ate-setup` | `hack/install-ate.sh` | -|---|---| -| `deploy ate-system` | `--deploy-ate-system` | -| `deploy ate-system --setup-csi=nfs` | `--deploy-ate-system --setup-csi=nfs` | -| `deploy atelet` | `--deploy-atelet` | -| `deploy apiserver` | `--deploy-ate-apiserver` | -| `deploy ate-controller` | (no shell equivalent) | -| `deploy atenet` | `--deploy-atenet` | -| `deploy podcertificate-controller` | (no shell equivalent) | -| `deploy sandboxconfig` | (no shell equivalent) | -| `deploy postgres` | (no shell equivalent) | +- `deploy ate-system` +- `deploy ate-system --setup-csi=nfs` +- `deploy atelet` +- `deploy apiserver` +- `deploy ate-controller` +- `deploy atenet` +- `deploy podcertificate-controller` +- `deploy sandboxconfig` +- `deploy postgres` `deploy ate-system` is the whole control plane: CRDs, RBAC, the store, the apiserver, the controller, atenet, and atelet. It creates every `create` @@ -122,11 +116,9 @@ a running cluster. ## Publish -| `ate-setup` | `hack/install-ate.sh` | -|---|---| -| `publish worker-images` | (no shell equivalent) | -| `publish release-images` | (no shell equivalent) | -| `publish release-binaries` | (no shell equivalent) | +- `publish worker-images` +- `publish release-images` +- `publish release-binaries` `publish worker-images` builds and pushes the worker images for the checked-out build and prints their refs; a WorkerPool points `spec.workerImage` @@ -144,11 +136,9 @@ attaches the files to the draft release. ## Delete -| `ate-setup` | `hack/install-ate.sh` | -|---|---| -| `delete ate-system` | `--delete-ate-system` | -| `delete atenet` | `--delete-atenet` | -| `delete all` | `--delete-all` | +- `delete ate-system` +- `delete atenet` +- `delete all` `delete all` removes every registered demo and then the control plane. @@ -156,73 +146,60 @@ attaches the files to the draft release. Individual secrets and config that `deploy ate-system` creates automatically. -| `ate-setup` | `hack/install-ate.sh` | -|---|---| -| `create jwt-authority-pool` | `--create-jwt-authority-pool-secret` | -| `create actor-id-ca-pool` | `--create-actor-id-ca-pool-secret` | -| `create actor-id-ca-certs` | `--create-actor-id-ca-certs-secret` | -| `create egress-mitm-ca-pool` | `--create-egress-mitm-ca-pool-secret` | -| `create podcertificate-controller-cas` | `--create-podcertificate-controller-cas` | -| `create api-server-env-vars` | `--create-api-server-env-vars` | -| `create api-authentication-config` | `--create-api-authentication-config` | -| `create api-config` | `--create-api-config` | +- `create jwt-authority-pool` +- `create actor-id-ca-pool` +- `create actor-id-ca-certs` +- `create egress-mitm-ca-pool` +- `create podcertificate-controller-cas` +- `create api-server-env-vars` +- `create api-authentication-config` +- `create api-config` ## Setup -| `ate-setup` | `hack/install-ate.sh` | -|---|---| -| `setup csi [driver]` | `--setup-csi[=DRIVER]` | +- `setup csi [driver]` -`driver` is one of `nfs`, `hostpath`, `both`, or `none`; `setup csi` with `none` as the default option. The hostpath is for KIND clusters only. NFS has no such +`driver` is one of `nfs`, `hostpath`, `both`, or `none`, and defaults to `nfs`. The hostpath is for KIND clusters only. NFS has no such restriction, but it does need the `nfsd` kernel module loaded on the nodes. ## Benchmarks -| `ate-setup` | `hack/install-ate.sh` | -|---|---| -| `deploy benchmarks` | `--deploy-benchmarks` | -| `delete benchmarks` | `--delete-benchmarks` | -| `--worker-count N` | `--benchmark-worker-count N` (default `1`) | -| `--sandbox-class gvisor\|microvm` | `--benchmark-sandbox-class CLASS` (default `gvisor`) | -| `BENCHMARK_ACTOR_MEMORY=SIZE` | `--benchmark-actor-memory SIZE` (default `256Mi`) | - -The memory limit has no flag: `benchmarking/workloads/deploy.sh` has always -taken it from the environment, and the shim exports it. - -The other two flags are per-command in `ate-setup` and global in -`hack/install-ate.sh`, which forwards them to whichever benchmark action runs. -See +- `deploy benchmarks` +- `delete benchmarks` +- `--worker-count N` (default `1`) +- `--sandbox-class gvisor|microvm` (default `gvisor`) +- `--benchmark-actor-memory SIZE` (default `256Mi`) + +`--benchmark-actor-memory` is a global flag (`BENCHMARK_ACTOR_MEMORY` when it +is absent); the other two are flags of the benchmark commands. See [`benchmarking/README.md`](../../benchmarking/README.md). ## Demos -| `ate-setup` | `hack/install-ate.sh` | +- `deploy demo NAME` +- `delete demo NAME` + +`NAME` is one of: + +| Command | Description | |---|---| -| `deploy demo NAME` | `--deploy-demo-NAME` | -| `delete demo NAME` | `--delete-demo-NAME` | - -`NAME` is the demo without its `demo-` prefix: - -| `ate-setup` | `hack/install-ate.sh` | Description | -|---|---|---| -| `deploy demo counter` | `--deploy-demo-counter` | A counter actor exercising snapshot, resume, and atenet ingress | -| `deploy demo counter --with-external-volume [--storage-class NAME]` | `--deploy-demo-counter-with-external-volume` (`STORAGE_CLASS=NAME`) | The same, plus an external volume and a pre-seeded file to validate (requires `--preview=ExternalVolumes`). Run `setup csi` first and name the class it created, e.g. `csi-nfs-sc`; defaults to `standard` | -| `deploy demo counter-microvm` | `--deploy-demo-counter-microvm` | The counter demo on microVM workers. Run `hack/install-microvm-deps.sh --install` first | -| `deploy demo egress` | `--deploy-demo-egress` | Egress policy enforcement through atenet; the actors trust the egress gateway's CA | -| `deploy demo egress-microvm` | `--deploy-demo-egress-microvm` | The same on microVM workers. Run `hack/install-microvm-deps.sh --install` first | -| `deploy demo jupyter` | `--deploy-demo-jupyter` | A Jupyter notebook server per actor, reached through atenet ingress | -| `deploy demo sandbox` | `--deploy-demo-sandbox` | An on-demand sandbox actor driven by the sandbox client | -| `deploy demo multi-template` | `--deploy-demo-multi-template` | Two ActorTemplates sharing one WorkerPool | -| `deploy demo parking` | `--deploy-demo-parking` | Actor parking and unparking on a small WorkerPool | -| `deploy demo claude-code-multiplex` | `--deploy-demo-claude-code-multiplex` | Several Claude Code agents multiplexed onto one WorkerPool (requires `ANTHROPIC_API_KEY`, `BUCKET_NAME`, `KO_DOCKER_REPO`) | - -Each has a matching `delete demo NAME` / `--delete-demo-NAME`. Demo flags bind +| `deploy demo counter` | A counter actor exercising snapshot, resume, and atenet ingress | +| `deploy demo counter --with-external-volume [--storage-class NAME]` | The same, plus an external volume and a pre-seeded file to validate (requires `--preview=ExternalVolumes`). Run `setup csi` first and name the class it created, e.g. `csi-nfs-sc`; defaults to `standard` | +| `deploy demo counter-microvm` | The counter demo on microVM workers. Run `hack/install-microvm-deps.sh --install` first | +| `deploy demo egress` | Egress policy enforcement through atenet; the actors trust the egress gateway's CA | +| `deploy demo egress-microvm` | The same on microVM workers. Run `hack/install-microvm-deps.sh --install` first | +| `deploy demo jupyter` | A Jupyter notebook server per actor, reached through atenet ingress | +| `deploy demo sandbox` | An on-demand sandbox actor driven by the sandbox client | +| `deploy demo multi-template` | Two ActorTemplates sharing one WorkerPool | +| `deploy demo parking` | Actor parking and unparking on a small WorkerPool | +| `deploy demo claude-code-multiplex` | Several Claude Code agents multiplexed onto one WorkerPool (requires `ANTHROPIC_API_KEY`, `KO_DOCKER_REPO`) | + +Each has a matching `delete demo NAME`. Demo flags bind to the deploy side only; teardown never reads them. The demo list is not hard-coded here — it is built from the registry in [`internal/demos`](internal/demos), one package per demo, so `go run ./cmd/ate-setup deploy demo --help` is authoritative for both the list and the per-demo flags. A new demo is a new package there, added to -[`internal/demos/all`](internal/demos/all) and mirrored into the `ATE_DEMOS` -list in `hack/install-ate.sh`, which a test keeps in step with the registry. +[`internal/demos/all`](internal/demos/all). diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 9c53e93872..d97a6bbdb7 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -3,17 +3,13 @@ `ate-setup` is a Go port of `hack/install-ate.sh`, the seven `hack/install-demo-*.sh` scripts it sourced, and `hack/setup-csi-*-kind.sh`. -The port has replaced them. The demo scripts are deleted, and -`hack/install-ate.sh` is a shim holding no install logic of its own: it -translates the flags and environment variables the installer has always -accepted onto `ate-setup` commands, so existing command lines keep working. See -[`commands.md`](commands.md) for the flag-by-flag mapping. - -This document covers what changed *behind* that mapping: the design decisions -of the port, against the shell it replaced. For what a user of -`hack/install-ate.sh` can observe after the switch, see -[`cli-diff.md`](cli-diff.md). For anything listed in neither, the port is -intended to be behavior-preserving. +The port has replaced them. `hack/install-ate.sh`, the demo scripts, and +`hack/setup-csi-hostpath-kind.sh` are deleted. See +[`commands.md`](commands.md) for the commands. + +This document covers the design decisions of the port, against the shell it +replaced. For anything not listed here, the port is intended to be +behavior-preserving. ## What is deliberately unchanged @@ -21,8 +17,7 @@ These were treated as contracts and reproduced exactly: - **Step log lines.** `log.Step` prints the same cyan `[step]: name` and the same step names (`deploy_ate_system`, `create_api_server_env_vars`, - `demo-counter_deploy`), so CI log scrapers keep working. Two lines lost a - parenthesized qualifier; see [`cli-diff.md`](cli-diff.md). + `demo-counter_deploy`), so CI log scrapers keep working. - **Manifest ordering.** Applying a directory is non-recursive and lexical, as `kubectl apply -f ` was. `delete_ate_system` depends on that ordering and the shell comments called out specific filename hazards, so @@ -58,8 +53,7 @@ These were treated as contracts and reproduced exactly: | Repository root | `git rev-parse --show-toplevel`, then `cd` | walk up for `go.mod`; no `chdir`, all paths absolute | The one-action-per-run change is the most visible: a line that passed -`--deploy-ate-system --deploy-demo-counter` becomes two `ate-setup` calls. That -is what the shim does with it, so the combined form keeps working. +`--deploy-ate-system --deploy-demo-counter` becomes two `ate-setup` calls. Invalid input now fails before any cluster mutation. `--atenet-dataplane=nginx` used to be caught by a pre-scan validation pass; `--worker-count 0` was not @@ -146,8 +140,9 @@ to publish the new image under the same repository and tag as the rest, and until it does, an install has to build it or take it from a registry that holds it. -Releases publish only `images.ReleaseComponents`, an allowlist: a component is -published only once it is added to it, so a new entry in `images.Components` +Releases publish `images.ReleaseComponents` and +`images.ReleaseDockerfileImages`, both allowlists: an image is published only +once it is added to one, so a new entry in `images.Components` stays out of releases until someone adds it on purpose. See [`docs/dev/releasing.md`](../../docs/dev/releasing.md#release-images) for what releases cover. Under `--image-repo`, a component that is not on the allowlist @@ -184,11 +179,12 @@ model, so NotFound is ignored — and so is a kind that no longer resolves, sinc teardown after the CRDs are gone must not fail. Beyond that, deletes are strict. -**Cluster profiles are new.** `--cluster-size` and `--cordon-control-plane` -postdate the shell installer, which the shim forwards them to. `size10` +**Cluster profiles are new.** `--cluster-size` and `--cordon-system-infra-components` +postdate the shell installer. `size10` renders the `podcert-size10` overlay and resizes the bundled PostgreSQL from -`postgres-size10/postgres-config-patch.yaml`; `--cordon-control-plane` -composes the `cordon-control-plane` component over every control plane apply +`postgres-size10/postgres-config-patch.yaml`; +`--cordon-system-infra-components` composes the +`cordon-system-infra-components` component over every system infra apply path. The size10 PostgreSQL changes are made to the decoded objects before the one server-side apply, not patched on afterwards: one rollout, the rollout wait sees the resized pod, and there is no second field manager for a later apply to @@ -258,8 +254,9 @@ through `ListActors`. The tolerances are preserved: no `ate-api-server`, or an a that cannot be reached, skips cleanup rather than failing, because `delete all` runs this for every demo. -**`docker buildx imagetools inspect` output is parsed as JSON**, not piped -through `jq -r '.manifest.digest'`. Build output still goes to stderr so the +**The pushed digest comes from `docker buildx build --metadata-file`**, not +from piping `docker buildx imagetools inspect` of the tag through +`jq -r '.manifest.digest'`. Build output still goes to stderr so the stdout/stderr split matches under CI log capture. ## Errors and diagnostics diff --git a/cmd/ate-setup/internal/cmd/csi_test.go b/cmd/ate-setup/internal/cmd/csi_test.go index 393a4dfc5d..106d67ece8 100644 --- a/cmd/ate-setup/internal/cmd/csi_test.go +++ b/cmd/ate-setup/internal/cmd/csi_test.go @@ -81,7 +81,7 @@ func TestAnUnknownCSIDriverIsRejectedBeforeAnythingRuns(t *testing.T) { // The accepted spellings have to make it past the same check, including the // bare `setup csi` that takes the default. func TestAcceptedCSIDriversPassArgumentValidation(t *testing.T) { - accepted := []string{"nfs", "hostpath", "both", "none", "true", "false", ""} + accepted := []string{"nfs", "hostpath", "both", "none"} t.Run("setup csi", func(t *testing.T) { for _, driver := range accepted { diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 76d3f12ff0..0d9f38c8eb 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -60,7 +60,7 @@ ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING or the ATE_API_POSTGRES_CLOUDSQL_* select an external database. Shape the install with the global --atenet-dataplane, --cluster-size, ---cordon-control-plane, and --fake-data-plane flags.`, +--cordon-system-infra-components, and --fake-data-plane flags.`, // Args runs before the root command loads the configuration, so only the // flag is readable here. Checking it keeps an unusable value from costing // a credential fetch, which is the common case; a value arriving from the diff --git a/cmd/ate-setup/internal/cmd/publish.go b/cmd/ate-setup/internal/cmd/publish.go index 5294157ef2..02df1ef07f 100644 --- a/cmd/ate-setup/internal/cmd/publish.go +++ b/cmd/ate-setup/internal/cmd/publish.go @@ -52,15 +52,15 @@ worker ref.`, var publishReleaseImagesCmd = &cobra.Command{ Use: "release-images", Short: "Build and push the release images, tagged with the release version", - Long: `Build and push the images a release publishes (images.ReleaseComponents) to -KO_DOCKER_REPO, all tagged with the build version, and print their pushed -references. The version comes from VERSION, else git describe, and has to be a -release tag (vMAJOR.MINOR.PATCH[-PRERELEASE]). + Long: `Build and push the images a release publishes (images.ReleaseComponents, and +images.ReleaseDockerfileImages, which need docker buildx) to KO_DOCKER_REPO, all +tagged with the build version, and print their pushed references. The version +comes from VERSION, else git describe, and has to be a release tag +(vMAJOR.MINOR.PATCH[-PRERELEASE]). VERSION=TAG ate-setup publish release-images --ko-docker-repo REPO -The release images are not a complete install; docs/dev/releasing.md lists -what they cover. +docs/dev/releasing.md lists what the release images cover. If REPO already holds every image for the tag on every platform, nothing is built unless --overwrite is set. After pushing, every image is checked for diff --git a/cmd/ate-setup/internal/cmd/root.go b/cmd/ate-setup/internal/cmd/root.go index 4fa51941ed..a33a97cf7b 100644 --- a/cmd/ate-setup/internal/cmd/root.go +++ b/cmd/ate-setup/internal/cmd/root.go @@ -216,10 +216,7 @@ func recordRun(cmd *cobra.Command, runErr error) { } } -// Root is the command tree, for callers that need to inspect it rather than -// run it. hack/install-ate.sh translates the historical installer flags into -// ate-setup command lines, and its test resolves each one against this to check -// it still names a command that exists. +// Root is the command tree, for tests that run it in-process. func Root() *cobra.Command { return rootCmd } // Execute runs the command tree. @@ -234,6 +231,11 @@ func Execute() { // ExecuteC reports which command ran, which the failure record needs and // Execute does not return. executed, err := rootCmd.ExecuteC() + // Cobra rejects an unknown command only at the root. Below it, a command + // that only groups others prints its help for any argument and succeeds. + if err == nil && !executed.Runnable() && executed.Flags().NArg() > 0 { + err = fmt.Errorf("unknown command %q for %q", executed.Flags().Arg(0), executed.CommandPath()) + } if err != nil { fmt.Fprintln(os.Stderr, "Error:", err) // After the error, so the retry line is the last thing on screen: diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index ab39897cf8..fda58cfa1e 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -25,6 +25,7 @@ import ( "net" "os" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -121,6 +122,10 @@ type Config struct { // owner on global:root (ATE_API_AUTHZ_BOOTSTRAP_OWNERS). APIAuthzBootstrapOwners string + // EgressGatewayExtraSPIFFEIDs hold ate-api-server's egress_gateway role + // next to atenet-egress (ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS). + EgressGatewayExtraSPIFFEIDs []string + // ActorJWTAlgorithm is the signing algorithm of the key in a new actor JWT // pool (ACTOR_JWT_ALGORITHM): ES256 or RS256. ActorJWTAlgorithm string @@ -129,14 +134,18 @@ type Config struct { // defaults to the in-cluster URL of the install's ate-idp-server Service. ActorJWTIssuer string - // BucketName is the snapshot bucket demos are templated with. + // BucketName is the object-storage bucket (BUCKET_NAME) passed to install + // scripts such as hack/install-microvm-deps.sh. The kind profile also + // derives SnapshotLocation from it. BucketName string // SnapshotLocation is ate-api-server's --snapshot-location // (SNAPSHOT_LOCATION): the cluster-wide object-storage URI actor snapshots // are stored under. Required unless the PerTemplateSnapshotLocation // preview gate is enabled, which also lets a - // template's snapshotConfig.storageLocation override it. + // template's snapshotConfig.storageLocation override it. The demos and + // benchmarks, and so e2e runs, need it even with the gate, since they do + // not set storageLocation. SnapshotLocation string // TransferGSAName is the name of the Google service account snapshot @@ -203,28 +212,34 @@ type Config struct { // or size10. ClusterSize string - // CordonControlPlane keeps the control plane off the worker nodes - // (ATE_INSTALL_CORDON_CONTROL_PLANE). It assumes a small shared pool - // labeled and tainted ate.dev/workloadType=ate-control-plane:NoSchedule, - // and a one-node pool labeled and tainted - // ate.dev/workloadType=ate-postgres:NoSchedule for postgres alone. - CordonControlPlane bool + // CordonSystemInfraComponents keeps the system infra components, such as + // ate-api-server, atenet-router, atenet-egress and postgres, off the + // worker nodes (ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS). It assumes a + // small shared pool labeled and tainted + // ate.dev/workloadType=ate-system-infra:NoSchedule, and a one-node pool + // labeled and tainted ate.dev/workloadType=ate-postgres:NoSchedule for + // postgres alone. + CordonSystemInfraComponents bool // FakeDataPlane installs fake-atelet in place of atelet and // fake-workersync in place of ate-controller // (ATE_INSTALL_FAKE_DATA_PLANE), for control-plane benchmarks. FakeDataPlane bool + // E2ETuning layers the e2e test kustomize component over control plane and + // data plane applies (ATE_INSTALL_E2E_TUNING). + E2ETuning bool + // IngressAllowedSPIFFEIDs are the SPIFFE IDs the ingress router lets // reach actors over its TLS listeners (ATE_INGRESS_ALLOWED_SPIFFE_IDS). - // See RequireIngressAllowedSPIFFEIDs. + // Empty means no client may reach actors through the router. IngressAllowedSPIFFEIDs []string // AdditionalEgressExtprocService is the optional NS/SVC:PORT external processor filter. AdditionalEgressExtprocService string // CredentialProviderJSON is the --credential-provider value; see - // CredentialProvider for its schema. + // CredentialProviders for its schema. CredentialProviderJSON string // AnthropicAPIKey is required only by the claude-code-multiplex demo. @@ -276,18 +291,6 @@ func (c *Config) Resolved() *Resolved { return c.resolved } -// RequireIngressAllowedSPIFFEIDs fails when the install would deploy the -// Envoy ingress router without an allowlist. That router authenticates -// clients on its TLS listeners, and refuses to start with no SPIFFE ID to -// match them against. The agentgateway router is configured statically and -// takes none. -func (c *Config) RequireIngressAllowedSPIFFEIDs() error { - if c.Router == RouterAgentgateway || len(c.IngressAllowedSPIFFEIDs) > 0 { - return nil - } - return c.Resolved().Require("ingressAllowedSPIFFEIDs") -} - // TransferServiceAccount is the email of the snapshot transfer service // account, built from TransferGSAName and ProjectID as tools/setup-gcp builds // it. It is empty when either is. @@ -435,10 +438,9 @@ func (c *Config) Path(parts ...string) string { return filepath.Join(append([]string{c.Root}, parts...)...) } -// kindUnsetVars are the GKE-specific variables the shell kind installer unset -// before delegating, so that a developer who had already sourced -// .ate-dev-env.sh into their shell did not end up pointing a local install at a -// cloud project. +// kindUnsetVars are the GKE-specific variables a Kind install drops, so that a +// developer who has already sourced .ate-dev-env.sh into their shell does not +// point a local install at a cloud project. var kindUnsetVars = []string{ "GCE_REGION", "CLUSTER_LOCATION", "NETWORK", "SUBNETWORK", "MEMORYSTORE_INSTANCE", "PROJECT_ID", @@ -508,18 +510,26 @@ func (c *Config) ScriptEnv() []string { if len(c.PreviewGates) > 0 { merged["ATE_PREVIEW"] = strings.Join(c.PreviewGates, ",") } - delete(merged, "ATE_INSTALL_CORDON_CONTROL_PLANE") - if c.CordonControlPlane { - merged["ATE_INSTALL_CORDON_CONTROL_PLANE"] = "true" + delete(merged, "ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS") + if c.CordonSystemInfraComponents { + merged["ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS"] = "true" } delete(merged, "ATE_INSTALL_FAKE_DATA_PLANE") if c.FakeDataPlane { merged["ATE_INSTALL_FAKE_DATA_PLANE"] = "true" } + delete(merged, "ATE_INSTALL_E2E_TUNING") + if c.E2ETuning { + merged["ATE_INSTALL_E2E_TUNING"] = "true" + } delete(merged, "ATE_INGRESS_ALLOWED_SPIFFE_IDS") if len(c.IngressAllowedSPIFFEIDs) > 0 { merged["ATE_INGRESS_ALLOWED_SPIFFE_IDS"] = strings.Join(c.IngressAllowedSPIFFEIDs, ",") } + delete(merged, "ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS") + if len(c.EgressGatewayExtraSPIFFEIDs) > 0 { + merged["ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS"] = strings.Join(c.EgressGatewayExtraSPIFFEIDs, ",") + } if c.AdditionalEgressExtprocService != "" { merged["ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE"] = c.AdditionalEgressExtprocService } @@ -572,7 +582,8 @@ const credentialProviderKey = "atenet.egress.credentialProvider" // credentialProviderUsage lists the accepted --credential-provider values. const credentialProviderUsage = `{"name":"k8s.io"} for the bundled Kubernetes Secrets provider, ` + `{"enabled":false} to turn egress credential injection off, ` + - `or {"name":"","address":":"} for a provider you deploy` + `or {"name":"","address":":"} for a provider you deploy; ` + + `a JSON array of provider objects selects several` // credentialProviderSpec is the schema of the --credential-provider JSON. type credentialProviderSpec struct { @@ -598,6 +609,11 @@ func (p CredentialProvider) Enabled() bool { return p.Name != "" } // Kubernetes reports whether the provider is the bundled one. func (p CredentialProvider) Kubernetes() bool { return p.Name == K8sCredentialProviderName } +// HasKubernetesProvider reports whether providers include the bundled one. +func HasKubernetesProvider(providers []CredentialProvider) bool { + return slices.ContainsFunc(providers, CredentialProvider.Kubernetes) +} + // ServerName is the SAN the gateway expects on the provider's serving // certificate: the address without its port. func (p CredentialProvider) ServerName() string { @@ -607,19 +623,10 @@ func (p CredentialProvider) ServerName() string { return p.Address } -// CredentialProvider parses --credential-provider, a JSON object that either +// credentialProvider parses a --credential-provider JSON object that either // turns injection off with enabled false or names a provider and its address. -// Only the bundled provider has a default address. A missing value is reported here rather than in validate -// because only the deploys that render the egress gateway need it. -func (c *Config) CredentialProvider() (CredentialProvider, error) { - raw := c.CredentialProviderJSON - if raw == "" { - // Every channel is named, not just the flag: a provider set in the - // configuration document is the documented way to do it, and a reader - // told only about the flag goes looking in the wrong place. - return CredentialProvider{}, fmt.Errorf("a credential provider is required; set one of %s: %s", - channelsFor(credentialProviderKey), credentialProviderUsage) - } +// Only the bundled provider has a default address. +func credentialProvider(raw string) (CredentialProvider, error) { invalid := func(format string, args ...any) (CredentialProvider, error) { return CredentialProvider{}, fmt.Errorf("invalid --credential-provider '%s': %s", raw, fmt.Sprintf(format, args...)) } @@ -657,3 +664,65 @@ func (c *Config) CredentialProvider() (CredentialProvider, error) { } return CredentialProvider{Name: spec.Name, Address: spec.Address}, nil } + +// CredentialProviders parses --credential-provider, which is one of: +// +// {"name":"k8s.io"} one provider +// [{"name":"k8s.io"},{"name":"","address":""}] several providers +// {"enabled":false} none: injection off +// +// credentialProvider checks each provider object. A missing value is reported +// here rather than in validate because only the deploys that render the egress +// gateway need it. +func (c *Config) CredentialProviders() ([]CredentialProvider, error) { + raw := c.CredentialProviderJSON + if raw == "" { + return nil, fmt.Errorf("a credential provider is required; set one of %s: %s", + channelsFor(credentialProviderKey), credentialProviderUsage) + } + invalid := func(format string, args ...any) ([]CredentialProvider, error) { + return nil, fmt.Errorf("invalid --credential-provider '%s': %s", raw, fmt.Sprintf(format, args...)) + } + var value any + if err := json.Unmarshal([]byte(raw), &value); err != nil { + return invalid("%v; want %s", err, credentialProviderUsage) + } + var elems []any + switch v := value.(type) { + case map[string]any: + p, err := credentialProvider(raw) + if err != nil || !p.Enabled() { + return nil, err + } + return []CredentialProvider{p}, nil + case []any: + elems = v + default: + return invalid("want a JSON object or an array of them: %s", credentialProviderUsage) + } + if len(elems) == 0 { + return invalid(`the array names no provider; {"enabled":false} turns injection off`) + } + var providers []CredentialProvider + for _, elem := range elems { + if _, ok := elem.(map[string]any); !ok { + return invalid("array entries must be JSON objects; want %s", credentialProviderUsage) + } + obj, err := json.Marshal(elem) + if err != nil { + return invalid("%v", err) + } + p, err := credentialProvider(string(obj)) + if err != nil { + return nil, err + } + if !p.Enabled() { + return invalid("enabled false is not allowed in an array") + } + if slices.ContainsFunc(providers, func(q CredentialProvider) bool { return q.Name == p.Name }) { + return invalid("provider name %q repeats", p.Name) + } + providers = append(providers, p) + } + return providers, nil +} diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 99ae080bbd..1fef3fed4f 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -61,11 +61,13 @@ func loadEnv(t *testing.T) { "ATE_API_POSTGRES_SERVER_CA_FILE", "ATE_ATENET_DATAPLANE", "ATE_CREDENTIAL_PROVIDER", + "ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS", "ATE_IMAGE_REPO", "ATE_IMAGE_TAG", "ATE_INGRESS_ALLOWED_SPIFFE_IDS", "ATE_INSTALL_CLUSTER_SIZE", - "ATE_INSTALL_CORDON_CONTROL_PLANE", + "ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS", + "ATE_INSTALL_E2E_TUNING", "ATE_INSTALL_FAKE_DATA_PLANE", "ATE_INSTALL_KIND", "ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER", @@ -129,8 +131,8 @@ func TestLoadDefaults(t *testing.T) { if cfg.ClusterSize != ClusterSizeSize0 { t.Errorf("ClusterSize = %q, want %q", cfg.ClusterSize, ClusterSizeSize0) } - if cfg.CordonControlPlane { - t.Error("CordonControlPlane = true, want false") + if cfg.CordonSystemInfraComponents { + t.Error("CordonSystemInfraComponents = true, want false") } } @@ -191,30 +193,30 @@ func TestLoadClusterSize(t *testing.T) { } } -func TestLoadCordonControlPlane(t *testing.T) { +func TestLoadCordonSystemInfraComponents(t *testing.T) { for _, tc := range []struct { name string opts Options env string want bool }{ - {name: "flag", opts: Options{CordonControlPlane: true}, want: true}, + {name: "flag", opts: Options{CordonSystemInfraComponents: true}, want: true}, {name: "environment true", env: "true", want: true}, {name: "environment false", env: "false", want: false}, } { t.Run(tc.name, func(t *testing.T) { loadEnv(t) - t.Setenv("ATE_INSTALL_CORDON_CONTROL_PLANE", tc.env) + t.Setenv("ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS", tc.env) cfg, err := Load(tc.opts) if err != nil { t.Fatalf("Load() error = %v", err) } - if cfg.CordonControlPlane != tc.want { - t.Errorf("CordonControlPlane = %v, want %v", cfg.CordonControlPlane, tc.want) + if cfg.CordonSystemInfraComponents != tc.want { + t.Errorf("CordonSystemInfraComponents = %v, want %v", cfg.CordonSystemInfraComponents, tc.want) } - _, exported := scriptEnvMap(t, cfg)["ATE_INSTALL_CORDON_CONTROL_PLANE"] + _, exported := scriptEnvMap(t, cfg)["ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS"] if exported != tc.want { - t.Errorf("ScriptEnv() exports ATE_INSTALL_CORDON_CONTROL_PLANE = %v, want %v", exported, tc.want) + t.Errorf("ScriptEnv() exports ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS = %v, want %v", exported, tc.want) } }) } @@ -248,6 +250,34 @@ func TestLoadFakeDataPlane(t *testing.T) { } } +func TestLoadE2ETuning(t *testing.T) { + for _, tc := range []struct { + name string + opts Options + env string + want bool + }{ + {name: "off by default"}, + {name: "flag", opts: Options{E2ETuning: true}, want: true}, + {name: "environment", env: "true", want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_INSTALL_E2E_TUNING", tc.env) + cfg, err := Load(tc.opts) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.E2ETuning != tc.want { + t.Errorf("E2ETuning = %v, want %v", cfg.E2ETuning, tc.want) + } + if _, exported := scriptEnvMap(t, cfg)["ATE_INSTALL_E2E_TUNING"]; exported != tc.want { + t.Errorf("ScriptEnv() exports ATE_INSTALL_E2E_TUNING = %v, want %v", exported, tc.want) + } + }) + } +} + func TestLoadIngressAllowedSPIFFEIDs(t *testing.T) { const a, b = "spiffe://cluster.local/ns/a/sa/a", "spiffe://cluster.local/ns/b/sa/b" for _, tc := range []struct { @@ -287,29 +317,39 @@ func TestLoadIngressAllowedSPIFFEIDs(t *testing.T) { } } -// Only the Envoy router authenticates clients against the allowlist, and it -// refuses to start without one. -func TestRequireIngressAllowedSPIFFEIDs(t *testing.T) { +func TestLoadEgressGatewayExtraSPIFFEIDs(t *testing.T) { + const a, b = "spiffe://cluster.local/ns/a/sa/a", "spiffe://cluster.local/ns/b/sa/b" for _, tc := range []struct { name string - cfg Config - wantErr bool + env string + want []string + wantErr string }{ - {name: "envoy without IDs", cfg: Config{Router: RouterEnvoy}, wantErr: true}, - {name: "envoy with IDs", cfg: Config{Router: RouterEnvoy, IngressAllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/a/sa/a"}}}, - {name: "agentgateway without IDs", cfg: Config{Router: RouterAgentgateway}}, + {name: "unset", want: nil}, + {name: "several, with stray separators", env: " " + a + ",," + b + ", ", want: []string{a, b}}, + {name: "not a SPIFFE ID", env: a + ",https://example.com/x", wantErr: "ateapi.authz.egressGatewayExtraSPIFFEIDs must be a comma-separated list of SPIFFE IDs"}, } { t.Run(tc.name, func(t *testing.T) { - err := tc.cfg.RequireIngressAllowedSPIFFEIDs() - if !tc.wantErr { - if err != nil { - t.Fatalf("RequireIngressAllowedSPIFFEIDs() = %v, want nil", err) + loadEnv(t) + if tc.env != "" { + t.Setenv("ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS", tc.env) + } + cfg, err := Load(Options{}) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("Load() error = %v, want one containing %q", err, tc.wantErr) } return } - var required *RequiredError - if !errors.As(err, &required) || required.Setting.Key != "ingressAllowedSPIFFEIDs" { - t.Fatalf("RequireIngressAllowedSPIFFEIDs() = %v, want a RequiredError for ingressAllowedSPIFFEIDs", err) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if !slices.Equal(cfg.EgressGatewayExtraSPIFFEIDs, tc.want) { + t.Errorf("EgressGatewayExtraSPIFFEIDs = %q, want %q", cfg.EgressGatewayExtraSPIFFEIDs, tc.want) + } + got, exported := scriptEnvMap(t, cfg)["ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS"] + if want := strings.Join(tc.want, ","); exported != (want != "") || got != want { + t.Errorf("ScriptEnv() ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS = %q (exported %v), want %q", got, exported, want) } }) } @@ -858,9 +898,8 @@ func TestLoadOtlpEndpoint(t *testing.T) { } } -// hack/install-ate-kind.sh exports ATE_INSTALL_KIND rather than passing a flag, -// so the environment has to select the Kind profile as completely as --kind -// does; a Kind install that only half-applied would push images to the wrong +// ATE_INSTALL_KIND has to select the Kind profile as completely as --kind does; +// a Kind install that only half-applied would push images to the wrong // registry. func TestLoadKindFromEnvironment(t *testing.T) { loadEnv(t) @@ -1020,6 +1059,10 @@ func TestLoadRejectsInvalidValues(t *testing.T) { {"kubernetes address without a port", Options{CredentialProvider: `{"name":"k8s.io","address":"secrets.ate-system.svc"}`}}, {"other provider without an address", Options{CredentialProvider: `{"name":"vault.example.com"}`}}, {"other provider address without a port", Options{CredentialProvider: `{"name":"vault.example.com","address":"vault.ate-system.svc"}`}}, + {"provider array empty", Options{CredentialProvider: `[]`}}, + {"provider array entry disabled", Options{CredentialProvider: `[{"enabled":false}]`}}, + {"provider array with a repeated name", Options{CredentialProvider: `[{"name":"k8s.io"},{"name":"k8s.io"}]`}}, + {"provider array entry without an address", Options{CredentialProvider: `[{"name":"k8s.io"},{"name":"vault.example.com"}]`}}, } { t.Run(tc.name, func(t *testing.T) { if _, err := Load(tc.opts); err == nil { @@ -1030,7 +1073,7 @@ func TestLoadRejectsInvalidValues(t *testing.T) { } // The provider selection is required only where the egress gateway is -// rendered, so Load accepts an absent value and CredentialProvider refuses it. +// rendered, so Load accepts an absent value and CredentialProviders refuses it. func TestCredentialProvider(t *testing.T) { loadEnv(t) @@ -1098,26 +1141,84 @@ func TestCredentialProvider(t *testing.T) { if err != nil { t.Fatalf("Load() error = %v", err) } - got, err := cfg.CredentialProvider() + got, err := cfg.CredentialProviders() if tc.wantErr { if err == nil { - t.Fatal("CredentialProvider() succeeded, want an error") + t.Fatal("CredentialProviders() succeeded, want an error") } return } if err != nil { - t.Fatalf("CredentialProvider() error = %v", err) + t.Fatalf("CredentialProviders() error = %v", err) } - if got != tc.want { - t.Errorf("CredentialProvider() = %+v, want %+v", got, tc.want) + var want []CredentialProvider + if tc.want.Enabled() { + want = []CredentialProvider{tc.want} } - if got.Enabled() != (tc.want.Name != "") { - t.Errorf("Enabled() = %v, want %v", got.Enabled(), tc.want.Name != "") + if !slices.Equal(got, want) { + t.Errorf("CredentialProviders() = %+v, want %+v", got, want) } }) } } +func TestCredentialProvidersArray(t *testing.T) { + loadEnv(t) + k8s := CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress} + for _, tc := range []struct { + name string + raw string + want []CredentialProvider + wantErr string + }{ + {name: "an array of one", raw: `[{"name":"k8s.io"}]`, want: []CredentialProvider{k8s}}, + { + name: "several providers", + raw: `[{"name":"k8s.io"},{"name":"sts.example.com","address":"sts.ate-system.svc:50051"}]`, + want: []CredentialProvider{k8s, {Name: "sts.example.com", Address: "sts.ate-system.svc:50051"}}, + }, + {name: "a null entry", raw: `[null]`, wantErr: "array entries must be JSON objects"}, + {name: "a string entry", raw: `["k8s.io"]`, wantErr: "array entries must be JSON objects"}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg, err := Load(Options{CredentialProvider: tc.raw}) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("Load() error = %v, want one containing %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("Load() error = %v", err) + } + got, err := cfg.CredentialProviders() + if err != nil { + t.Fatalf("CredentialProviders() error = %v", err) + } + if !slices.Equal(got, tc.want) { + t.Errorf("CredentialProviders() = %+v, want %+v", got, tc.want) + } + }) + } +} + +func TestHasKubernetesProvider(t *testing.T) { + k8s := CredentialProvider{Name: K8sCredentialProviderName} + vault := CredentialProvider{Name: "vault.example.com"} + for _, tc := range []struct { + providers []CredentialProvider + want bool + }{ + {nil, false}, + {[]CredentialProvider{vault}, false}, + {[]CredentialProvider{vault, k8s}, true}, + } { + if got := HasKubernetesProvider(tc.providers); got != tc.want { + t.Errorf("HasKubernetesProvider(%v) = %v, want %v", tc.providers, got, tc.want) + } + } +} + func TestCredentialProviderServerName(t *testing.T) { for addr, want := range map[string]string{ "k8s-credential-provider.ate-system.svc:50051": "k8s-credential-provider.ate-system.svc", @@ -1177,8 +1278,8 @@ func TestLoadEnvoyConcurrency(t *testing.T) { } } -// --kind must reproduce the exports in the shell kind installer, including -// clearing the GKE coordinates and isolating the snapshot bucket name. +// --kind must apply the whole Kind profile, including clearing the GKE +// coordinates and isolating the snapshot bucket name. func TestLoadKindProfile(t *testing.T) { loadEnv(t) t.Setenv("PROJECT_ID", "some-gke-project") diff --git a/cmd/ate-setup/internal/config/coverage_test.go b/cmd/ate-setup/internal/config/coverage_test.go index 8f23e55ea5..5585769f7a 100644 --- a/cmd/ate-setup/internal/config/coverage_test.go +++ b/cmd/ate-setup/internal/config/coverage_test.go @@ -90,7 +90,7 @@ func valueFor(key string) string { return RouterEnvoy case "clusterSize": return ClusterSizeSize0 - case "ingressAllowedSPIFFEIDs": + case "ingressAllowedSPIFFEIDs", "ateapi.authz.egressGatewayExtraSPIFFEIDs": return "spiffe://cluster.local/ns/a/sa/b" case "ateapi.postgres.cloudsql.ipType": return CloudSQLIPTypePSC diff --git a/cmd/ate-setup/internal/config/errors_test.go b/cmd/ate-setup/internal/config/errors_test.go index aacc43f296..0e59c8349f 100644 --- a/cmd/ate-setup/internal/config/errors_test.go +++ b/cmd/ate-setup/internal/config/errors_test.go @@ -248,9 +248,9 @@ func TestValidationErrorsNameTheSupplyingChannel(t *testing.T) { func TestCredentialProviderErrorNamesTheFileChannel(t *testing.T) { useFixtures(t) cfg := &Config{} - _, err := cfg.CredentialProvider() + _, err := cfg.CredentialProviders() if err == nil { - t.Fatal("CredentialProvider() with nothing set returned no error") + t.Fatal("CredentialProviders() with nothing set returned no error") } for _, want := range []string{ "--credential-provider", diff --git a/cmd/ate-setup/internal/config/file.go b/cmd/ate-setup/internal/config/file.go index 542628e7a0..683f94a8a5 100644 --- a/cmd/ate-setup/internal/config/file.go +++ b/cmd/ate-setup/internal/config/file.go @@ -61,8 +61,7 @@ func ParseFile(path string) (*File, error) { raw, err := os.ReadFile(path) if err != nil { // Name the absolute path: a relative one means different files from - // different working directories, and the shell installer changes to - // the repository root before running. + // different working directories. abs, absErr := filepath.Abs(path) if absErr != nil { abs = path diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index a85f7d64d5..d585b17bbb 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -31,10 +31,6 @@ import ( ) // ConfigPathEnv names the configuration document when --config is not given. -// -// It exists so the file channel crosses hack/install-ate.sh, which forwards -// only a fixed set of flags but passes the whole environment to the child -// process. Every other setting reaches ate-setup that way already. const ConfigPathEnv = "ATE_CONFIG" // LoadOptions carries the inputs that select where configuration comes from, @@ -85,11 +81,9 @@ func LoadFlags(fs *pflag.FlagSet, o LoadOptions) (*Config, error) { // configPathFrom picks the configuration document, flag over environment. // -// A relative path resolves against the working directory, which is the -// repository root when the shell installer invoked this: it does cd "$ROOT" -// before running the binary. ParseFile reports the absolute path it tried, so -// a path that meant something else from the caller's directory is visible in -// the error rather than silently missing. +// A relative path resolves against the working directory. ParseFile reports +// the absolute path it tried, so a path that meant something else from the +// caller's directory is visible in the error rather than silently missing. func configPathFrom(o LoadOptions) string { if o.ConfigPath != "" { return o.ConfigPath @@ -143,9 +137,8 @@ func resolveKind(fs *pflag.FlagSet, file *File, env map[string]string) bool { // mergeDevEnv layers .ate-dev-env.sh underneath the process environment. // -// Sourcing is skipped for Kind installs the same way the shell kind installer -// exports NO_DEV_ENV: the GKE-shaped variables in a developer's file would -// otherwise point a local install at a cloud project. +// Sourcing is skipped for Kind installs: the GKE-shaped variables in a +// developer's file would otherwise point a local install at a cloud project. func mergeDevEnv(root string, env map[string]string, noDevEnv, kind bool) error { if noDevEnv || kind || os.Getenv("NO_DEV_ENV") != "" { return nil @@ -230,7 +223,8 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro ClusterLocation: r.String("gcp.clusterLocation"), ExpectedJWTIssuer: r.String("ateapi.expectedJWTIssuer"), - APIAuthzBootstrapOwners: bootstrapOwners, + APIAuthzBootstrapOwners: bootstrapOwners, + EgressGatewayExtraSPIFFEIDs: splitList(r.String("ateapi.authz.egressGatewayExtraSPIFFEIDs")), BucketName: r.String("storage.bucketName"), SnapshotLocation: r.String("storage.snapshotLocation"), @@ -273,8 +267,9 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro PodcertWorkersPerSigner: podcertWorkers, EnvoyConcurrency: envoyConcurrency, ClusterSize: r.String("clusterSize"), - CordonControlPlane: r.Bool("cordonControlPlane"), + CordonSystemInfraComponents: r.Bool("cordonSystemInfraComponents"), FakeDataPlane: r.Bool("fakeDataPlane.enabled"), + E2ETuning: r.Bool("e2e.tuning"), IngressAllowedSPIFFEIDs: splitList(r.String("ingressAllowedSPIFFEIDs")), AdditionalEgressExtprocService: r.String("atenet.egress.additionalExtprocService"), CredentialProviderJSON: r.String("atenet.egress.credentialProvider"), @@ -412,14 +407,20 @@ func validateResolved(cfg *Config, r *Resolved) error { return &InvalidError{Value: size, Want: ClusterSizeSize0 + " or " + ClusterSizeSize10} } - // The router rejects a malformed ID at startup; catch it before the - // rollout does. + // The router and ate-api-server reject a malformed ID at startup; catch it + // before the rollout does. for _, id := range cfg.IngressAllowedSPIFFEIDs { if _, err := spiffeid.FromString(id); err != nil { ids, _ := r.Value("ingressAllowedSPIFFEIDs") return &InvalidError{Value: ids, Want: fmt.Sprintf("a comma-separated list of SPIFFE IDs (%q: %v)", id, err)} } } + for _, id := range cfg.EgressGatewayExtraSPIFFEIDs { + if _, err := spiffeid.FromString(id); err != nil { + ids, _ := r.Value("ateapi.authz.egressGatewayExtraSPIFFEIDs") + return &InvalidError{Value: ids, Want: fmt.Sprintf("a comma-separated list of SPIFFE IDs (%q: %v)", id, err)} + } + } switch cfg.APILogLevel { case "", "debug", "info", "warn", "error": @@ -458,7 +459,7 @@ func validateResolved(cfg *Config, r *Resolved) error { // configuration rather than part-way through a deploy. Whether one is // required at all is the deploy's question, not this one's. if cfg.CredentialProviderJSON != "" { - if _, err := cfg.CredentialProvider(); err != nil { + if _, err := cfg.CredentialProviders(); err != nil { // The parser's own message says which part is wrong and what to // write instead; only the channel is added. Replacing it with the // generic "must be " would drop the reason. diff --git a/cmd/ate-setup/internal/config/load_test.go b/cmd/ate-setup/internal/config/load_test.go index b48f8b3ebe..0ac2c83c29 100644 --- a/cmd/ate-setup/internal/config/load_test.go +++ b/cmd/ate-setup/internal/config/load_test.go @@ -128,9 +128,8 @@ func TestLoadFlagsRejectsBadConfigPath(t *testing.T) { } } -// The file channel must cross hack/install-ate.sh, which forwards only a fixed -// set of flags but passes the whole environment through. ATE_CONFIG is how it -// gets there without a shim change. +// ATE_CONFIG names the document when --config is absent, and the flag +// outranks it. func TestConfigPathFromEnvironment(t *testing.T) { loadEnv(t) path := filepath.Join(t.TempDir(), "install.yaml") @@ -191,9 +190,8 @@ func TestConfigPathFromEnvironment(t *testing.T) { }) } -// A relative path means different files from different working directories, -// and the shell installer changes to the repository root. The error names the -// absolute path so that is visible rather than silent. +// A relative path means different files from different working directories. +// The error names the absolute path so that is visible rather than silent. func TestMissingConfigErrorNamesAbsolutePath(t *testing.T) { loadEnv(t) t.Setenv(ConfigPathEnv, "no-such-config.yaml") diff --git a/cmd/ate-setup/internal/config/options_test.go b/cmd/ate-setup/internal/config/options_test.go index 3e8fba4ffc..9f99a2545f 100644 --- a/cmd/ate-setup/internal/config/options_test.go +++ b/cmd/ate-setup/internal/config/options_test.go @@ -38,8 +38,9 @@ type Options struct { PodcertWorkersPerSigner int EnvoyConcurrency int ClusterSize string - CordonControlPlane bool + CordonSystemInfraComponents bool FakeDataPlane bool + E2ETuning bool AdditionalEgressExtprocService string CredentialProvider string OtlpEndpoint string @@ -103,8 +104,9 @@ func optionsFlagSet(opts Options) (*pflag.FlagSet, error) { val bool }{ {"kind", opts.Kind}, - {"cordon-control-plane", opts.CordonControlPlane}, + {"cordon-system-infra-components", opts.CordonSystemInfraComponents}, {"fake-data-plane", opts.FakeDataPlane}, + {"e2e-tuning", opts.E2ETuning}, } { if kv.val { if err := fs.Set(kv.flag, "true"); err != nil { diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 389da4d55c..9d9d2f4d13 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -147,7 +147,7 @@ var Registry = []Setting{ { Key: "storage.bucketName", Env: "BUCKET_NAME", Flag: "storage-bucket-name", Kind: KindString, - Usage: "Snapshot bucket the demos are templated with", + Usage: "Object-storage bucket passed to install scripts; the kind profile derives --snapshot-location from it", }, { Key: "storage.snapshotLocation", Env: "SNAPSHOT_LOCATION", Flag: "snapshot-location", Kind: KindString, @@ -188,9 +188,16 @@ var Registry = []Setting{ "Unset means kubectl-ate's ServiceAccount, system:serviceaccount::ate-client", }, { - // The egress gateway's credential provider, as a JSON object. Upstream - // takes it as one value on one flag; it is carried the same way here - // so the two channels accept the same text. + Key: "ateapi.authz.egressGatewayExtraSPIFFEIDs", Env: "ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS", + Flag: "egress-gateway-extra-spiffe-ids", Kind: KindString, + Usage: "Comma-separated SPIFFE IDs that hold ate-api-server's egress_gateway role next to atenet-egress, " + + "such as a credential provider that mints actor JWTs. The role also reads actors and mints actor " + + "certificates. The podidentity signer issues spiffe://cluster.local/ns//sa/", + }, + { + // The egress gateway's credential providers: one JSON object, or a JSON array + // of them for several. A repeated flag would be ideal, but a Setting holds one + // string on every channel, so an array is the least worst option. Key: "atenet.egress.credentialProvider", Env: "ATE_CREDENTIAL_PROVIDER", Flag: "credential-provider", Kind: KindString, Usage: "Credential provider the egress gateway injects credentials from, as JSON; " + @@ -269,6 +276,11 @@ var Registry = []Setting{ Flag: "podcert-workers-per-signer", Kind: KindInt, Default: "0", Usage: "Worker goroutines per signer in podcertificate-controller", }, + { + Key: "e2e.tuning", Env: "ATE_INSTALL_E2E_TUNING", Flag: "e2e-tuning", + Kind: KindBool, Default: "false", + Usage: "Apply the e2e test component (short atelet snapshot delays for end-to-end test runs)", + }, { Key: "e2e.envoyConcurrency", Env: "E2E_ATE_ENVOY_CONCURRENCY", Flag: "e2e-envoy-concurrency", Kind: KindInt, Default: "0", @@ -286,10 +298,10 @@ var Registry = []Setting{ Usage: "Coarse-grained sizing of Substrate: size0 or size10", }, { - Key: "cordonControlPlane", Env: "ATE_INSTALL_CORDON_CONTROL_PLANE", Flag: "cordon-control-plane", + Key: "cordonSystemInfraComponents", Env: "ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS", Flag: "cordon-system-infra-components", Kind: KindBool, Default: "false", - Usage: "Keep the control plane off the worker nodes. Assumes a small shared pool labeled " + - "and tainted ate.dev/workloadType=ate-control-plane:NoSchedule, and a one-node pool " + + Usage: "Keep the system infra components off the worker nodes. Assumes a small shared pool labeled " + + "and tainted ate.dev/workloadType=ate-system-infra:NoSchedule, and a one-node pool " + "labeled and tainted ate.dev/workloadType=ate-postgres:NoSchedule for postgres alone", }, { @@ -306,8 +318,8 @@ var Registry = []Setting{ Key: "ingressAllowedSPIFFEIDs", Env: "ATE_INGRESS_ALLOWED_SPIFFE_IDS", Flag: "ingress-allowed-spiffe-ids", Kind: KindString, Usage: "SPIFFE IDs (exact match, comma-separated) of the clients the ingress router lets reach actors; " + - "any listed client may reach any actor. Required to deploy the router with " + - "--atenet-dataplane=envoy. The podidentity signer issues spiffe://cluster.local/ns//sa/", + "any listed client may reach any actor. Unset, no client may reach actors through the router. " + + "The podidentity signer issues spiffe://cluster.local/ns//sa/", }, { Key: "otlpEndpoint", Env: "ATE_OTLP_ENDPOINT", Flag: "otlp-endpoint", Kind: KindString, diff --git a/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex.go b/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex.go index f8b7db51b3..b89587f23e 100644 --- a/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex.go +++ b/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex.go @@ -53,7 +53,7 @@ func init() { d := &demo{} d.sub = demos.Substrate{ DemoName: "demo-claude-code-multiplex", - Short: "Several Claude Code agents multiplexed onto one WorkerPool (requires ANTHROPIC_API_KEY, BUCKET_NAME, KO_DOCKER_REPO)", + Short: "Several Claude Code agents multiplexed onto one WorkerPool (requires ANTHROPIC_API_KEY, KO_DOCKER_REPO)", WorkerPoolManifest: poolManifest, Deployments: []steps.TemplateRef{{Atespace: atespace, Name: "claude-workerpool"}}, Templates: agentTemplates(), @@ -85,9 +85,6 @@ func (d *demo) Deploy(ctx context.Context, e *steps.Env) error { if e.Cfg.AnthropicAPIKey == "" { return fmt.Errorf("ANTHROPIC_API_KEY must be set") } - if e.Cfg.BucketName == "" { - return fmt.Errorf("BUCKET_NAME must be set") - } if e.Cfg.KODockerRepo == "" { return fmt.Errorf("KO_DOCKER_REPO must be set (see hack/ate-dev-env.sh.example)") } diff --git a/cmd/ate-setup/internal/demos/demotest/demotest.go b/cmd/ate-setup/internal/demos/demotest/demotest.go index 5e845ba967..aeef03a372 100644 --- a/cmd/ate-setup/internal/demos/demotest/demotest.go +++ b/cmd/ate-setup/internal/demos/demotest/demotest.go @@ -48,7 +48,7 @@ func Env(t *testing.T) *steps.Env { if err != nil { t.Fatalf("Resolve: %v", err) } - cfg := &config.Config{Root: root, BucketName: "ate-snapshots"} + cfg := &config.Config{Root: root} cfg.SetResolved(r) return &steps.Env{Cfg: cfg} } @@ -85,6 +85,10 @@ func AssertRenderedActorTemplate(t *testing.T, manifest []byte, want resources.A if got := resources.ActorTemplateRefFromActorTemplate(template); got != want { t.Errorf("manifest names template %s, want %s", got, want) } + // The API rejects storageLocation without its preview gate. + if got := template.GetSnapshotConfig().GetStorageLocation(); got != "" { + t.Errorf("manifest sets storageLocation %q, want none", got) + } } // assertNoPlaceholders checks that no non-comment line still contains a diff --git a/cmd/ate-setup/internal/demos/render.go b/cmd/ate-setup/internal/demos/render.go index a755057dfe..2d12137879 100644 --- a/cmd/ate-setup/internal/demos/render.go +++ b/cmd/ate-setup/internal/demos/render.go @@ -19,10 +19,6 @@ import ( "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) -// bucketNamePlaceholder is substituted into every demo template with the -// snapshot bucket for this environment. -const bucketNamePlaceholder = "BUCKET_NAME" - // ExternalVolumePlaceholders are the optional external-volume hooks in the // counter templates. Every path except // `deploy demo counter --with-external-volume` drops them, which removes the @@ -33,13 +29,7 @@ var ExternalVolumePlaceholders = []string{ "EXTERNAL_VOLUMES", } -// Render expands a demo template with the configured bucket name. -func Render(e *steps.Env, relPath string, extraValues map[string]string, drop []string) ([]byte, error) { - values := map[string]string{ - bucketNamePlaceholder: e.Cfg.BucketName, - } - for k, v := range extraValues { - values[k] = v - } +// Render expands a demo template with values. +func Render(e *steps.Env, relPath string, values map[string]string, drop []string) ([]byte, error) { return render.Template(e.Cfg.Path(relPath), values, drop) } diff --git a/cmd/ate-setup/internal/demos/substrate.go b/cmd/ate-setup/internal/demos/substrate.go index e5812d847b..6278efa2b2 100644 --- a/cmd/ate-setup/internal/demos/substrate.go +++ b/cmd/ate-setup/internal/demos/substrate.go @@ -68,10 +68,10 @@ type Substrate struct { // is a cloud-hypervisor cold boot plus a checkpoint, on nested KVM in CI. GoldenTimeout time.Duration - // RenderValues optionally supplies extra placeholder values at deploy - // time, for demos whose manifests need more than ${BUCKET_NAME} (e.g. a - // freshly built image digest). Delete renders only the pool manifest, so - // deploy-only values may live solely here. + // RenderValues optionally supplies placeholder values at deploy time, for + // demos whose manifests need them (e.g. a freshly built image digest). + // Delete renders only the pool manifest, so deploy-only values may live + // solely here. RenderValues func(ctx context.Context, e *steps.Env) (map[string]string, error) } diff --git a/cmd/ate-setup/internal/images/dockerfile.go b/cmd/ate-setup/internal/images/dockerfile.go index 78211a3c3c..a55df1c773 100644 --- a/cmd/ate-setup/internal/images/dockerfile.go +++ b/cmd/ate-setup/internal/images/dockerfile.go @@ -15,7 +15,6 @@ package images import ( - "bytes" "context" "encoding/json" "errors" @@ -69,13 +68,29 @@ func dockerfilePlatforms(rootDir, koDefaultPlatforms string) (string, error) { } // dockerBuildArgs returns the docker arguments that build contextPath for -// platforms and push it as tag. extraFlags go before the context. -func dockerBuildArgs(platforms string, extraFlags []string, tag, contextPath string) []string { - args := []string{"buildx", "build", "--platform=" + platforms, "--push"} +// platforms, push it as tag, and write the build metadata to metadataFile. +// extraFlags go before the context. +func dockerBuildArgs(platforms string, extraFlags []string, tag, metadataFile, contextPath string) []string { + args := []string{"buildx", "build", "--platform=" + platforms, "--push", "--metadata-file", metadataFile} args = append(args, extraFlags...) return append(args, "-t", tag, contextPath) } +// pushedDigest returns the digest buildx pushed, from the file written by +// --metadata-file. +func pushedDigest(metadata []byte) (string, error) { + var parsed struct { + Digest string `json:"containerimage.digest"` + } + if err := json.Unmarshal(metadata, &parsed); err != nil { + return "", fmt.Errorf("while parsing the buildx metadata: %w", err) + } + if parsed.Digest == "" { + return "", errors.New("the buildx metadata has no containerimage.digest") + } + return parsed.Digest, nil +} + // BuildDockerfileImage builds a Dockerfile-based image from contextPath, pushes // it to dockerRepo/, and returns the digest-pinned reference. // extraFlags are passed to docker buildx build, e.g. --cache-from/--cache-to. @@ -84,15 +99,30 @@ func dockerBuildArgs(platforms string, extraFlags []string, tag, contextPath str // push to; the returned reference always uses the digest, so a stale tag can // never be resolved by accident. func BuildDockerfileImage(ctx context.Context, rootDir, dockerRepo, imageName, contextPath, koDefaultPlatforms string, extraFlags []string) (string, error) { + return PublishDockerfileImage(ctx, rootDir, dockerRepo, imageName, fmt.Sprintf("build-%d", time.Now().Unix()), contextPath, koDefaultPlatforms, extraFlags) +} + +// PublishDockerfileImage is BuildDockerfileImage with the tag chosen by the +// caller, for a release that publishes every image under one tag. +func PublishDockerfileImage(ctx context.Context, rootDir, dockerRepo, imageName, tag, contextPath, koDefaultPlatforms string, extraFlags []string) (string, error) { repo := strings.TrimSuffix(dockerRepo, "/") + "/" + imageName - stageTag := fmt.Sprintf("%s:build-%d", repo, time.Now().Unix()) + stageTag := repo + ":" + tag platforms, err := dockerfilePlatforms(rootDir, koDefaultPlatforms) if err != nil { return "", err } + metadata, err := os.CreateTemp("", "buildx-metadata-*.json") + if err != nil { + return "", err + } + if err := metadata.Close(); err != nil { + return "", err + } + defer os.Remove(metadata.Name()) + build := exec.CommandContext(ctx, "docker", - dockerBuildArgs(platforms, extraFlags, stageTag, contextPath)...) + dockerBuildArgs(platforms, extraFlags, stageTag, metadata.Name(), contextPath)...) build.Dir = rootDir // The shell version sent build output to stderr so it could capture the // image reference on stdout; keeping that split makes the two behave the @@ -103,26 +133,13 @@ func BuildDockerfileImage(ctx context.Context, rootDir, dockerRepo, imageName, c return "", fmt.Errorf("while building the %s image: %w", imageName, err) } - inspect := exec.CommandContext(ctx, "docker", "buildx", "imagetools", "inspect", - stageTag, "--format", "{{json .}}") - inspect.Dir = rootDir - inspect.Stderr = os.Stderr - var out bytes.Buffer - inspect.Stdout = &out - if err := inspect.Run(); err != nil { - return "", fmt.Errorf("while inspecting %s: %w", stageTag, err) - } - - var inspected struct { - Manifest struct { - Digest string `json:"digest"` - } `json:"manifest"` - } - if err := json.Unmarshal(out.Bytes(), &inspected); err != nil { - return "", fmt.Errorf("while parsing the image manifest of %s: %w", stageTag, err) + raw, err := os.ReadFile(metadata.Name()) + if err != nil { + return "", err } - if inspected.Manifest.Digest == "" { - return "", fmt.Errorf("failed to resolve the image digest from %s", stageTag) + digest, err := pushedDigest(raw) + if err != nil { + return "", fmt.Errorf("%s: %w", stageTag, err) } - return repo + "@" + inspected.Manifest.Digest, nil + return repo + "@" + digest, nil } diff --git a/cmd/ate-setup/internal/images/dockerfile_test.go b/cmd/ate-setup/internal/images/dockerfile_test.go index 97fea6f08f..1bf04315c3 100644 --- a/cmd/ate-setup/internal/images/dockerfile_test.go +++ b/cmd/ate-setup/internal/images/dockerfile_test.go @@ -23,18 +23,48 @@ import ( ) func TestDockerBuildArgs(t *testing.T) { - got := dockerBuildArgs("linux/amd64", []string{"--cache-from", "type=gha"}, "repo/img:build-1", "ctx") - want := []string{"buildx", "build", "--platform=linux/amd64", "--push", "--cache-from", "type=gha", "-t", "repo/img:build-1", "ctx"} + got := dockerBuildArgs("linux/amd64", []string{"--cache-from", "type=gha"}, "repo/img:build-1", "meta.json", "ctx") + want := []string{"buildx", "build", "--platform=linux/amd64", "--push", "--metadata-file", "meta.json", "--cache-from", "type=gha", "-t", "repo/img:build-1", "ctx"} if !slices.Equal(got, want) { t.Errorf("dockerBuildArgs() = %q, want %q", got, want) } - got = dockerBuildArgs("linux/amd64", nil, "repo/img:build-1", "ctx") - want = []string{"buildx", "build", "--platform=linux/amd64", "--push", "-t", "repo/img:build-1", "ctx"} + got = dockerBuildArgs("linux/amd64", nil, "repo/img:build-1", "meta.json", "ctx") + want = []string{"buildx", "build", "--platform=linux/amd64", "--push", "--metadata-file", "meta.json", "-t", "repo/img:build-1", "ctx"} if !slices.Equal(got, want) { t.Errorf("dockerBuildArgs() without flags = %q, want %q", got, want) } } +// The digest comes from what buildx pushed, so a tag that names another push +// cannot stand in for it. +func TestPushedDigest(t *testing.T) { + const digest = "sha256:bab239626dc054856e882a2ef022dfa845834113d213697e1dccd27dfbfac1a5" + for _, tc := range []struct { + name string + metadata string + want string + wantErr bool + }{ + { + name: "multi-platform push", + metadata: `{"buildx.build.ref":"ate-release/ate-release0/x","containerimage.digest":"` + digest + `","image.name":"localhost:5000/substrate/envoy-dataplane:v1"}`, + want: digest, + }, + {name: "no digest", metadata: `{"buildx.build.ref":"ate-release/ate-release0/x"}`, wantErr: true}, + {name: "not JSON", metadata: ``, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := pushedDigest([]byte(tc.metadata)) + if (err != nil) != tc.wantErr { + t.Fatalf("pushedDigest() error = %v, want error %v", err, tc.wantErr) + } + if got != tc.want { + t.Errorf("pushedDigest() = %q, want %q", got, tc.want) + } + }) + } +} + // ko builds for .ko.yaml's defaultPlatforms unless KO_DEFAULTPLATFORMS says // otherwise, and a Dockerfile image installed beside the ko images has to // cover the same platforms or it cannot run on some of their nodes. diff --git a/cmd/ate-setup/internal/images/images.go b/cmd/ate-setup/internal/images/images.go index 2110fe26b9..99151a405d 100644 --- a/cmd/ate-setup/internal/images/images.go +++ b/cmd/ate-setup/internal/images/images.go @@ -72,6 +72,7 @@ var released = []string{ "cmd/atenet", "cmd/ateworker-gvisor", "cmd/ateworker-microvm", + "cmd/credential-provider/kubernetes-secrets", "cmd/objectstore-plugin", "cmd/podcertcontroller", } @@ -79,6 +80,25 @@ var released = []string{ // ReleaseComponents returns the Components a release publishes. func ReleaseComponents() []string { return slices.Clone(released) } +// DockerfileImage is an image built from a Dockerfile with docker buildx +// rather than with ko. +type DockerfileImage struct { + // Name is the image name under the repository. + Name string + // Context is the build context, relative to the repository root. + Context string +} + +// EnvoyDataplane is the image the envoy router and egress gateways run. +var EnvoyDataplane = DockerfileImage{Name: "envoy-dataplane", Context: "cmd/dataplane/envoy"} + +// releasedDockerfiles is the allowlist of DockerfileImages a release +// publishes, the counterpart of released. +var releasedDockerfiles = []DockerfileImage{EnvoyDataplane} + +// ReleaseDockerfileImages returns the DockerfileImages a release publishes. +func ReleaseDockerfileImages() []DockerfileImage { return slices.Clone(releasedDockerfiles) } + // IsReleased reports whether a release publishes pkg. func IsReleased(pkg string) bool { return slices.Contains(released, pkg) } diff --git a/cmd/ate-setup/internal/images/prebuilt_test.go b/cmd/ate-setup/internal/images/prebuilt_test.go index ccf5bc48ce..7fa604d3ba 100644 --- a/cmd/ate-setup/internal/images/prebuilt_test.go +++ b/cmd/ate-setup/internal/images/prebuilt_test.go @@ -232,15 +232,15 @@ func TestPrebuiltReportsEveryFailure(t *testing.T) { // such as one the operator published to. When the lookup fails, the error says // releases do not publish it; for a released image it does not. func TestPrebuiltUnreleasedImages(t *testing.T) { - const ref = "image: ko://go.ate.dev/substrate/cmd/credential-provider/kubernetes-secrets\n" - const hint = "releases do not publish kubernetes-secrets" + const ref = "image: ko://go.ate.dev/substrate/demos/counter\n" + const hint = "releases do not publish counter" got, err := images.NewPrebuilt(testSource, newStubRegistry().digest). ResolveBytes(context.Background(), []byte(ref)) if err != nil { t.Fatalf("ResolveBytes() from a registry that holds it = %v", err) } - if want := "image: example.com/substrate/kubernetes-secrets:v1.2.3" + digestSuffix + "\n"; string(got) != want { + if want := "image: example.com/substrate/counter:v1.2.3" + digestSuffix + "\n"; string(got) != want { t.Errorf("ResolveBytes() = %q, want %q", got, want) } diff --git a/cmd/ate-setup/internal/images/release.go b/cmd/ate-setup/internal/images/release.go index 72fc7f9d95..75d30f1213 100644 --- a/cmd/ate-setup/internal/images/release.go +++ b/cmd/ate-setup/internal/images/release.go @@ -42,6 +42,11 @@ const SourceURL = "https://github.com/agent-substrate/substrate" // from. const RevisionLabel = "org.opencontainers.image.revision" +// VersionLabel is the OCI image label naming the release an image belongs to. +// It replaces any value inherited from a base image, such as the Ubuntu +// release envoyproxy/envoy carries. +const VersionLabel = "org.opencontainers.image.version" + // releaseTag matches a release tag, vMAJOR.MINOR.PATCH, or a release candidate, // vMAJOR.MINOR.PATCH-PRERELEASE. hack/release/draft-release.sh accepts the same. var releaseTag = regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$`) diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 481f72de1e..acb50a3667 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -187,7 +187,9 @@ func sharedBootstrapOwnerWarning(bootstrapOwners, namespace string) string { // snapshots. It is used whether or not the PerTemplateSnapshotLocation preview // gate is enabled; with the gate, an actor template's // snapshotConfig.storageLocation overrides it. It may be empty only when the -// gate is enabled, in which case every template must set its own. +// gate is enabled. CreateActorTemplate then fails with FailedPrecondition for +// a template without storageLocation, which includes every demo and +// benchmark. func clusterSnapshotLocation(location string, gates []string) (string, error) { if location != "" { if err := resources.ValidateSnapshotLocation(location); err != nil { @@ -312,7 +314,7 @@ func (e *Env) EnsureEnvVarsSafeStandalone(ctx context.Context) error { } return fmt.Errorf("the running ate-api-server Deployment does not reference the %s Secret; "+ "rewriting the env vars alone would leave it without a DSN on its next restart. "+ - "Run `ate-setup deploy ate-apiserver` instead, which also updates the Deployment", SecretAPIEnvVars) + "Run `ate-setup deploy apiserver` instead, which also updates the Deployment", SecretAPIEnvVars) } // annotateAPIServerEnvHash stamps the running Deployment's pod template with @@ -355,6 +357,9 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er if err := setAPIServerAuthzArgs(objs, e.Cfg.APIAuthzBootstrapOwners); err != nil { return err } + if err := setAPIServerEgressGatewayArgs(objs, e.Namespace(), e.Cfg.EgressGatewayExtraSPIFFEIDs); err != nil { + return err + } if msg := sharedBootstrapOwnerWarning(e.Cfg.APIAuthzBootstrapOwners, e.Namespace()); msg != "" { log.Warnf("%s", msg) } @@ -365,9 +370,40 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er const apiServerAuthzFlag = "--authz-bootstrap-owners" // setAPIServerAuthzArgs sets ate-api-server's bootstrap owners flag on its -// container in objs, replacing any the manifest already carries. A changed -// flag changes the pod template, so it rolls the Deployment. +// container in objs, replacing any the manifest already carries. func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, bootstrapOwners string) error { + var values []string + if strings.Trim(bootstrapOwners, " ,") != "" { + values = []string{bootstrapOwners} + } + return setAPIServerFlag(objs, apiServerAuthzFlag, values) +} + +// apiServerEgressGatewayFlag is the ate-api-server flag +// setAPIServerEgressGatewayArgs owns. +const apiServerEgressGatewayFlag = "--egress-gateway-spiffe-id" + +// setAPIServerEgressGatewayArgs passes atenet-egress's SPIFFE ID and then each +// of extras, without duplicates, to ate-api-server's egress gateway flag. With +// no extras it drops the flag, leaving ate-api-server's default. +func setAPIServerEgressGatewayArgs(objs []*unstructured.Unstructured, namespace string, extras []string) error { + var ids []string + if len(extras) > 0 { + // Passing the flag replaces ate-api-server's default. + ids = []string{installdefaults.EgressSPIFFEID(namespace)} + for _, id := range extras { + if !slices.Contains(ids, id) { + ids = append(ids, id) + } + } + } + return setAPIServerFlag(objs, apiServerEgressGatewayFlag, ids) +} + +// setAPIServerFlag replaces every flag arg on ate-api-server's container in +// objs with one flag=value arg per value, in order. A changed flag changes the +// pod template, so it rolls the Deployment. +func setAPIServerFlag(objs []*unstructured.Unstructured, flag string, values []string) error { var dep *unstructured.Unstructured for _, obj := range objs { if obj.GetKind() == "Deployment" && obj.GetName() == "ate-api-server" { @@ -376,7 +412,7 @@ func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, bootstrapOwners st } } if dep == nil { - return fmt.Errorf("the manifest has no deployment/ate-api-server to set authorization flags on") + return fmt.Errorf("the manifest has no deployment/ate-api-server to set %s on", flag) } containers, found, err := unstructured.NestedSlice(dep.Object, "spec", "template", "spec", "containers") if err != nil || !found { @@ -392,10 +428,10 @@ func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, bootstrapOwners st return fmt.Errorf("%s: %w", kube.Describe(dep), err) } args = slices.DeleteFunc(args, func(arg string) bool { - return arg == apiServerAuthzFlag || strings.HasPrefix(arg, apiServerAuthzFlag+"=") + return arg == flag || strings.HasPrefix(arg, flag+"=") }) - if strings.Trim(bootstrapOwners, " ,") != "" { - args = append(args, apiServerAuthzFlag+"="+bootstrapOwners) + for _, v := range values { + args = append(args, flag+"="+v) } if err := unstructured.SetNestedStringSlice(container, args, "args"); err != nil { return fmt.Errorf("%s: %w", kube.Describe(dep), err) diff --git a/cmd/ate-setup/internal/steps/apienv_test.go b/cmd/ate-setup/internal/steps/apienv_test.go index 3c97dea3dc..db6694a706 100644 --- a/cmd/ate-setup/internal/steps/apienv_test.go +++ b/cmd/ate-setup/internal/steps/apienv_test.go @@ -695,6 +695,63 @@ spec: } } +func TestSetAPIServerEgressGatewayArgs(t *testing.T) { + const manifest = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: ate-api-server + namespace: ate-system +spec: + template: + spec: + containers: + - name: ate-api-server + args: + - --config-file=/run/apiconfig.ate.dev/apiconfig.textproto + - --egress-gateway-spiffe-id=stale +` + const ( + egress = "spiffe://cluster.local/ns/ate-system/sa/atenet-egress" + a = "spiffe://cluster.local/ns/ate-system/sa/sts-credential-provider" + b = "spiffe://cluster.local/ns/plugins/sa/other" + ) + for _, tc := range []struct { + name string + extras []string + want []string + }{ + { + name: "gateway first, then extras without duplicates", + extras: []string{a, egress, b, a}, + want: []string{ + "--config-file=/run/apiconfig.ate.dev/apiconfig.textproto", + "--egress-gateway-spiffe-id=" + egress, + "--egress-gateway-spiffe-id=" + a, + "--egress-gateway-spiffe-id=" + b, + }, + }, + { + name: "no extras drops the flag", + want: []string{"--config-file=/run/apiconfig.ate.dev/apiconfig.textproto"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + objs, err := kube.DecodeManifestBytes([]byte(manifest)) + if err != nil { + t.Fatalf("DecodeManifestBytes() error = %v", err) + } + if err := setAPIServerEgressGatewayArgs(objs, "ate-system", tc.extras); err != nil { + t.Fatalf("setAPIServerEgressGatewayArgs() error = %v", err) + } + containers, _, _ := unstructured.NestedSlice(objs[0].Object, "spec", "template", "spec", "containers") + args, _, _ := unstructured.NestedStringSlice(containers[0].(map[string]any), "args") + if diff := cmp.Diff(tc.want, args); diff != "" { + t.Errorf("ate-api-server args (-want +got):\n%s", diff) + } + }) + } +} + func TestCreateAPIServerEnvVarsOmitsAuthz(t *testing.T) { e := &Env{Cfg: &config.Config{ TransferGSAName: testTransferGSAName, ProjectID: testProjectID, diff --git a/cmd/ate-setup/internal/steps/create.go b/cmd/ate-setup/internal/steps/create.go index 705365b25e..296a450cb5 100644 --- a/cmd/ate-setup/internal/steps/create.go +++ b/cmd/ate-setup/internal/steps/create.go @@ -63,10 +63,12 @@ func (e *Env) CreateJWTAuthorityPoolSecret(ctx context.Context) error { return e.createJWTPool(ctx, e.Namespace(), SecretActorIDJWTPool) } -// CreateActorIDCAPoolSecret generates the actor-identity CA pool. +// CreateActorIDCAPoolSecret generates the actor-identity CA pool. External +// relying parties trust this root, so it uses ECDSA P-256, which is more +// broadly accepted than Ed25519. func (e *Env) CreateActorIDCAPoolSecret(ctx context.Context) error { log.Step("create_actor_id_ca_pool_secret") - return e.createCAPool(ctx, e.Namespace(), SecretActorIDCAPool) + return e.createCAPool(ctx, e.Namespace(), SecretActorIDCAPool, localca.KeyTypeECDSAP256) } // CreateEgressMITMCAPoolSecret generates the egress MITM CA pool. @@ -102,13 +104,13 @@ func (e *Env) CreatePodCertificateControllerCAs(ctx context.Context) error { if err := e.Kube.EnsureNamespace(ctx, NamespacePodCert); err != nil { return err } - if err := e.createCAPool(ctx, NamespacePodCert, SecretServiceDNSCA); err != nil { + if err := e.createCAPool(ctx, NamespacePodCert, SecretServiceDNSCA, localca.KeyTypeED25519); err != nil { return err } - if err := e.createCAPool(ctx, NamespacePodCert, SecretPodIdentityCA); err != nil { + if err := e.createCAPool(ctx, NamespacePodCert, SecretPodIdentityCA, localca.KeyTypeED25519); err != nil { return err } - return e.createCAPool(ctx, NamespacePodCert, SecretPostgresCA) + return e.createCAPool(ctx, NamespacePodCert, SecretPostgresCA, localca.KeyTypeED25519) } // CreateActorIDCACertsSecret derives a certificate-only trust bundle from the @@ -238,7 +240,7 @@ func buildAuthenticationConfig(issuer string) string { // An existing pool is left alone. Regenerating it would rotate the root out // from under every certificate already issued from it, and the callers in the // shell script all guarded these calls with an existence check for that reason. -func (e *Env) createCAPool(ctx context.Context, namespace, name string) error { +func (e *Env) createCAPool(ctx context.Context, namespace, name string, keyType localca.KeyType) error { exists, err := e.Kube.SecretExists(ctx, namespace, name) if err != nil { return err @@ -248,7 +250,7 @@ func (e *Env) createCAPool(ctx context.Context, namespace, name string) error { return nil } - data, err := newCAPoolSecretData(poolKeyID, localca.KeyTypeED25519) + data, err := newCAPoolSecretData(poolKeyID, keyType) if err != nil { return fmt.Errorf("while generating the CA pool for %s/%s: %w", namespace, name, err) } diff --git a/cmd/ate-setup/internal/steps/create_test.go b/cmd/ate-setup/internal/steps/create_test.go index 0cb3ece7e8..64ca78fb53 100644 --- a/cmd/ate-setup/internal/steps/create_test.go +++ b/cmd/ate-setup/internal/steps/create_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/internal/localca" "go.ate.dev/substrate/internal/localjwtauthority" @@ -112,7 +113,8 @@ func TestNewCAPoolSecretData(t *testing.T) { keyType localca.KeyType wantAlg x509.PublicKeyAlgorithm }{ - {"actor identity", poolKeyID, localca.KeyTypeED25519, x509.Ed25519}, + {"pod certificates", poolKeyID, localca.KeyTypeED25519, x509.Ed25519}, + {"actor identity", poolKeyID, localca.KeyTypeECDSAP256, x509.ECDSA}, {"egress mitm", poolKeyID, localca.KeyTypeECDSAP256, x509.ECDSA}, } { t.Run(tc.name, func(t *testing.T) { @@ -165,6 +167,27 @@ func TestNewCAPoolSecretData(t *testing.T) { } } +// External relying parties trust the actor identity root, and many accept +// ECDSA but not Ed25519. +func TestCreateActorIDCAPoolSecretIsECDSA(t *testing.T) { + ctx := context.Background() + e := &Env{Cfg: &config.Config{}, Kube: fakeKube(t)} + if err := e.CreateActorIDCAPoolSecret(ctx); err != nil { + t.Fatalf("CreateActorIDCAPoolSecret() error = %v", err) + } + secret, err := e.Kube.Typed.CoreV1().Secrets(e.Namespace()).Get(ctx, SecretActorIDCAPool, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Get(%s) error = %v", SecretActorIDCAPool, err) + } + pool, err := localca.Unmarshal(secret.Data["pool"]) + if err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if got := pool.CAs[0].RootCertificate.PublicKeyAlgorithm; got != x509.ECDSA { + t.Errorf("root key algorithm = %v, want %v", got, x509.ECDSA) + } +} + func TestNewJWTPoolSecretData(t *testing.T) { for _, alg := range []string{"ES256", "RS256"} { t.Run(alg, func(t *testing.T) { diff --git a/cmd/ate-setup/internal/steps/csi.go b/cmd/ate-setup/internal/steps/csi.go index f42183257b..f93b8bc779 100644 --- a/cmd/ate-setup/internal/steps/csi.go +++ b/cmd/ate-setup/internal/steps/csi.go @@ -69,17 +69,12 @@ type csiDriverSpec struct { // installsNothing reports whether the value selects no driver at all. func (s csiDriverSpec) installsNothing() bool { return !s.hostpath && !s.nfs } -// csiDrivers is every accepted --setup-csi value. "true", "false" and the empty -// string are the shell installer's boolean spellings, from before the flag grew -// driver names, and still work. +// csiDrivers is every accepted --setup-csi value. var csiDrivers = map[string]csiDriverSpec{ "nfs": {nfs: true}, "hostpath": {hostpath: true}, "both": {hostpath: true, nfs: true}, - "true": {hostpath: true, nfs: true}, "none": {}, - "false": {}, - "": {}, } // ValidateCSIDriver reports whether driver names something --setup-csi accepts. diff --git a/cmd/ate-setup/internal/steps/csi_test.go b/cmd/ate-setup/internal/steps/csi_test.go index 695593f140..21b56e4c9b 100644 --- a/cmd/ate-setup/internal/steps/csi_test.go +++ b/cmd/ate-setup/internal/steps/csi_test.go @@ -59,12 +59,10 @@ func TestSetupCSIDecidesBeforeTouchingTheCluster(t *testing.T) { error string }{ {name: "none", driver: "none"}, - {name: "the shell installer's false", driver: "false"}, - {name: "unset", driver: ""}, {name: "hostpath off Kind", driver: "hostpath", error: "only supported for Kind"}, {name: "both off Kind", driver: "both", error: "only supported for Kind"}, - {name: "the shell installer's true off Kind", driver: "true", error: "only supported for Kind"}, {name: "unknown driver", driver: "nfsv4", error: `unknown CSI driver "nfsv4"`}, + {name: "the shell installer's true", driver: "true", error: `unknown CSI driver "true"`}, } { t.Run(tc.name, func(t *testing.T) { e := &Env{Cfg: &config.Config{Kind: tc.kind}} @@ -170,9 +168,7 @@ func TestTheNFSSetupsApplyTheSharedStorageClass(t *testing.T) { } } -// Both spellings of each driver selection have to keep selecting the same -// thing: --setup-csi was a boolean in the shell installer, and the scripts and -// dev-env files that still pass true or false have to keep working. +// Each --setup-csi value selects the drivers its name says. func TestCSIDriverSpellings(t *testing.T) { for _, tc := range []struct { driver string @@ -181,10 +177,7 @@ func TestCSIDriverSpellings(t *testing.T) { {"nfs", csiDriverSpec{nfs: true}}, {"hostpath", csiDriverSpec{hostpath: true}}, {"both", csiDriverSpec{hostpath: true, nfs: true}}, - {"true", csiDriverSpec{hostpath: true, nfs: true}}, {"none", csiDriverSpec{}}, - {"false", csiDriverSpec{}}, - {"", csiDriverSpec{}}, } { t.Run(tc.driver, func(t *testing.T) { got, ok := csiDrivers[tc.driver] diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index e235e96b0a..68dba51c63 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) @@ -64,15 +65,11 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { } // Fail fast on an unusable credential provider selection before touching // the cluster. - provider, err := e.Cfg.CredentialProvider() + providers, err := e.Cfg.CredentialProviders() if err != nil { return err } - // Likewise a router that would refuse to start without an allowlist, and - // a credential exchange without a transfer service account. - if err := e.Cfg.RequireIngressAllowedSPIFFEIDs(); err != nil { - return err - } + // Likewise a credential exchange without a transfer service account. if err := e.Cfg.RequireTransferServiceAccount(); err != nil { return err } @@ -163,7 +160,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { } // After the podcertificate controller: the provider serves with a // projected pod certificate. - if provider.Kubernetes() { + if config.HasKubernetesProvider(providers) { err = e.deployK8sCredentialProvider(ctx) } else { err = e.removeK8sCredentialProvider(ctx) @@ -171,7 +168,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err != nil { return err } - if err := e.applyAtenetEgress(ctx, provider); err != nil { + if err := e.applyAtenetEgress(ctx, providers); err != nil { return err } @@ -186,7 +183,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { rollout{kube.KindDeployment, "atenet-egress"}, rollout{kube.KindDaemonSet, "atelet"}, ) - if provider.Kubernetes() { + if config.HasKubernetesProvider(providers) { waits = append(waits, rollout{kube.KindDeployment, k8sCredentialProviderDeployment}) } for _, w := range waits { @@ -534,13 +531,10 @@ func (e *Env) ateletManifestPath() string { func (e *Env) DeployAtenet(ctx context.Context) error { log.Step("deploy_atenet") - provider, err := e.Cfg.CredentialProvider() + providers, err := e.Cfg.CredentialProviders() if err != nil { return err } - if err := e.Cfg.RequireIngressAllowedSPIFFEIDs(); err != nil { - return err - } if err := e.EnsureCRDs(ctx); err != nil { return err } @@ -564,7 +558,7 @@ func (e *Env) DeployAtenet(ctx context.Context) error { if err := e.EnsureEgressMITMCAPoolSecret(ctx); err != nil { return err } - if provider.Kubernetes() { + if config.HasKubernetesProvider(providers) { err = e.deployK8sCredentialProvider(ctx) } else { err = e.removeK8sCredentialProvider(ctx) @@ -572,12 +566,12 @@ func (e *Env) DeployAtenet(ctx context.Context) error { if err != nil { return err } - if err := e.applyAtenetEgress(ctx, provider); err != nil { + if err := e.applyAtenetEgress(ctx, providers); err != nil { return err } deployments := []string{"atenet-router", "atenet-egress"} - if provider.Kubernetes() { + if config.HasKubernetesProvider(providers) { deployments = append(deployments, k8sCredentialProviderDeployment) } for _, name := range deployments { diff --git a/cmd/ate-setup/internal/steps/env.go b/cmd/ate-setup/internal/steps/env.go index 6af68a5c06..26552ca1e1 100644 --- a/cmd/ate-setup/internal/steps/env.go +++ b/cmd/ate-setup/internal/steps/env.go @@ -172,16 +172,16 @@ func (e *Env) koRunner() (*ko.Runner, error) { // KO_DOCKER_REPO. A pre-built install builds nothing: --image-repo installs // images someone else published, and needs neither docker nor a registry to // push to, so the image is pinned from REPO/name:TAG like every ko image. -// That only works if someone published it there: `ate-setup publish -// release-images` does not, so an install from release images fails here. -func (e *Env) dockerfileImage(ctx context.Context, name, contextDir string) (string, error) { - if ref, ok := e.dockerfileImages[name]; ok { +// That only works if the repository holds it and can be read: releases +// publish it, but older releases and other mirrors may not. +func (e *Env) dockerfileImage(ctx context.Context, img images.DockerfileImage) (string, error) { + if ref, ok := e.dockerfileImages[img.Name]; ok { return ref, nil } var ref string var err error if !e.Cfg.Images.IsPrebuilt() { - ref, err = images.BuildDockerfileImage(ctx, e.Cfg.Root, e.Cfg.KODockerRepo, name, e.Cfg.Path(contextDir), e.Cfg.KODefaultPlatforms, e.Cfg.DockerBuildFlags) + ref, err = images.BuildDockerfileImage(ctx, e.Cfg.Root, e.Cfg.KODockerRepo, img.Name, e.Cfg.Path(img.Context), e.Cfg.KODefaultPlatforms, e.Cfg.DockerBuildFlags) } else { resolver, rErr := e.imageResolver() if rErr != nil { @@ -191,11 +191,11 @@ func (e *Env) dockerfileImage(ctx context.Context, name, contextDir string) (str if !ok { return "", fmt.Errorf("image resolver is %T, not a pre-built resolver", resolver) } - ref, err = prebuilt.Pin(ctx, name) + ref, err = prebuilt.Pin(ctx, img.Name) if err != nil { - return "", fmt.Errorf("%s is built from a Dockerfile, and releases do not publish it; "+ + return "", fmt.Errorf("%s is built from a Dockerfile, and the image repository has no readable copy; "+ "build it from source by dropping --image-repo, or choose --atenet-dataplane=agentgateway, "+ - "which does not use it: %w", name, err) + "which does not use it: %w", img.Name, err) } } if err != nil { @@ -204,7 +204,7 @@ func (e *Env) dockerfileImage(ctx context.Context, name, contextDir string) (str if e.dockerfileImages == nil { e.dockerfileImages = make(map[string]string) } - e.dockerfileImages[name] = ref + e.dockerfileImages[img.Name] = ref return ref, nil } diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 8a2d6d3e36..1cf30003c0 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -17,6 +17,7 @@ package steps import ( "bytes" "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -25,6 +26,7 @@ import ( "time" "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" "go.ate.dev/substrate/cmd/ate-setup/internal/kustomize" "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) @@ -32,33 +34,44 @@ import ( const ( // installDir is the manifest root, relative to the repository root. installDir = "manifests/ate-install" - // Envoy dataplane image name - envoyDataplaneImage = "envoy-dataplane" - // Envoy dataplane Dockerfile path - envoyDataplaneDockefile = "cmd/dataplane/envoy" ) -// cordonControlPlaneComponent is the kustomize component that pins each -// control plane workload to its own node, layered over every control plane -// apply under --cordon-control-plane. -const cordonControlPlaneComponent = installDir + "/components/cordon-control-plane" +// cordonSystemInfraComponent is the kustomize component that pins each +// system infra workload to its own node pool, layered over every system +// infra apply under --cordon-system-infra-components. +const cordonSystemInfraComponent = installDir + "/components/cordon-system-infra-components" + +// e2eComponent is the kustomize component that tunes the control plane and +// data plane for end-to-end test runs, layered over every control plane apply +// under --e2e-tuning. +const e2eComponent = installDir + "/components/e2e" // fakeDataPlaneComponent is the kustomize component that puts fake-atelet in // atelet's DaemonSet and fake-workersync in place of ate-controller, layered // over the atelet and ate-controller applies under --fake-data-plane. const fakeDataPlaneComponent = installDir + "/components/fake-data-plane" +const agentgatewayStaticMTLSComponent = installDir + "/components/agentgateway-static-mtls" + // components lists the kustomize components every control plane apply is // composed with. Each patch targets its workloads by name, and kustomize leaves // a stream alone when nothing in it matches, so composing a manifest that // carries none of them is harmless. func (e *Env) components() []string { var components []string + // e2e must come before fake-data-plane: e2e appends to atelet's args, + // and fake-data-plane removes args entirely. + if e.Cfg.E2ETuning { + components = append(components, e.Cfg.Path(e2eComponent)) + } if e.Cfg.FakeDataPlane { components = append(components, e.Cfg.Path(fakeDataPlaneComponent)) } - if e.Cfg.CordonControlPlane { - components = append(components, e.Cfg.Path(cordonControlPlaneComponent)) + if e.Cfg.Router == config.RouterAgentgateway { + components = append(components, e.Cfg.Path(agentgatewayStaticMTLSComponent)) + } + if e.Cfg.CordonSystemInfraComponents { + components = append(components, e.Cfg.Path(cordonSystemInfraComponent)) } return components } @@ -142,12 +155,15 @@ func (e *Env) renderSystemManifests(ctx context.Context) ([]byte, error) { if manifest, err = setIDPServerIssuer(manifest, e.Cfg.ActorJWTIssuer); err != nil { return nil, err } + if manifest, err = e.setAgentgatewayIngressAllowedSPIFFEIDs(manifest); err != nil { + return nil, err + } if e.Cfg.Router != config.RouterAgentgateway { manifest, err = e.patchAtenetRouterAllowedSPIFFEIDs(manifest) if err != nil { return nil, err } - imageReference, err := e.dockerfileImage(ctx, envoyDataplaneImage, envoyDataplaneDockefile) + imageReference, err := e.dockerfileImage(ctx, images.EnvoyDataplane) if err != nil { return nil, err } @@ -160,7 +176,15 @@ func (e *Env) renderSystemManifests(ctx context.Context) ([]byte, error) { // selected dataplane. func (e *Env) renderAtenetRouterManifest(ctx context.Context) ([]byte, error) { if e.Cfg.Router == config.RouterAgentgateway { - return e.renderResolve(ctx, e.Cfg.Path(installDir+"/agentgateway-router")) + manifest, err := e.render(e.Cfg.Path(installDir + "/agentgateway-router")) + if err != nil { + return nil, err + } + manifest, err = e.setAgentgatewayIngressAllowedSPIFFEIDs(manifest) + if err != nil { + return nil, err + } + return e.ResolveManifestBytes(ctx, manifest) } manifest, err := e.render(e.Cfg.Manifest("atenet-router.yaml")) if err != nil { @@ -170,7 +194,7 @@ func (e *Env) renderAtenetRouterManifest(ctx context.Context) ([]byte, error) { if err != nil { return nil, err } - imageReference, err := e.dockerfileImage(ctx, envoyDataplaneImage, envoyDataplaneDockefile) + imageReference, err := e.dockerfileImage(ctx, images.EnvoyDataplane) if err != nil { return nil, err } @@ -178,13 +202,33 @@ func (e *Env) renderAtenetRouterManifest(ctx context.Context) ([]byte, error) { return e.ResolveManifestBytes(ctx, manifest) } +func (e *Env) setAgentgatewayIngressAllowedSPIFFEIDs(raw []byte) ([]byte, error) { + if e.Cfg.Router != config.RouterAgentgateway { + return raw, nil + } + // nil would marshal as null, which agentgateway's networkAuthorization CEL rule can't evaluate. + allowed := e.Cfg.IngressAllowedSPIFFEIDs + if allowed == nil { + allowed = []string{} + } + ids, err := json.Marshal(allowed) + if err != nil { + return nil, fmt.Errorf("encoding AgentGateway ingress SPIFFE IDs: %w", err) + } + marker := []byte("[ATE_INGRESS_ALLOWED_SPIFFE_IDS]") + if count := bytes.Count(raw, marker); count != 1 { + return nil, fmt.Errorf("expected 1 AgentGateway ingress SPIFFE IDs placeholder, found %d", count) + } + return bytes.Replace(raw, marker, ids, 1), nil +} + // atenetEgressManifestPath returns the envoy egress gateway manifest. func (e *Env) atenetEgressManifestPath() string { return e.Cfg.Manifest("atenet-egress.yaml") } // renderAtenetEgressManifest produces the atenet egress manifest. -func (e *Env) renderAtenetEgressManifest(ctx context.Context, provider config.CredentialProvider) ([]byte, error) { +func (e *Env) renderAtenetEgressManifest(ctx context.Context, providers []config.CredentialProvider) ([]byte, error) { general := e.Cfg.AdditionalEgressExtprocService != "" if e.Cfg.Router == config.RouterAgentgateway { @@ -198,14 +242,18 @@ func (e *Env) renderAtenetEgressManifest(ctx context.Context, provider config.Cr if err != nil { return nil, err } - raw, err = patchAgentgatewayEgressInject(raw, provider) + raw, err = e.setAgentgatewayIngressAllowedSPIFFEIDs(raw) + if err != nil { + return nil, err + } + raw, err = patchAgentgatewayEgressInject(raw, providers) if err != nil { return nil, err } return e.ResolveManifestBytes(ctx, raw) } - imageReference, err := e.dockerfileImage(ctx, envoyDataplaneImage, envoyDataplaneDockefile) + imageReference, err := e.dockerfileImage(ctx, images.EnvoyDataplane) if err != nil { return nil, err } @@ -223,7 +271,7 @@ func (e *Env) renderAtenetEgressManifest(ctx context.Context, provider config.Cr if err != nil { return nil, err } - raw, err = e.patchAtenetEgressInject(raw, provider) + raw, err = e.patchAtenetEgressInject(raw, providers) if err != nil { return nil, err } @@ -250,10 +298,10 @@ func patchAtenetEgressEnvoyConcurrency(raw []byte, concurrency int) ([]byte, err // patchAtenetRouterAllowedSPIFFEIDs appends --ingress-allowed-spiffe-ids to the // atenet-router container's args in the Envoy router Deployment. The base // manifest carries no allowlist, because which clients may reach actors is the -// install's to decide. +// install's to decide. Without one the router serves no traffic. func (e *Env) patchAtenetRouterAllowedSPIFFEIDs(raw []byte) ([]byte, error) { - if err := e.Cfg.RequireIngressAllowedSPIFFEIDs(); err != nil { - return nil, err + if len(e.Cfg.IngressAllowedSPIFFEIDs) == 0 { + return raw, nil } return appendContainerArgs(raw, "atenet-router", "atenet-router", "--ingress-allowed-spiffe-ids="+strings.Join(e.Cfg.IngressAllowedSPIFFEIDs, ",")) @@ -312,19 +360,22 @@ func (e *Env) patchEnvoyDataplaneImage(raw []byte, imageRef string) []byte { // egress sidecar's args with the credential-provider flags, or removes it when // injection is off, which leaves the gateway with no provider. It takes the // manifest bytes so it can run after the general patch. -func (e *Env) patchAtenetEgressInject(raw []byte, provider config.CredentialProvider) ([]byte, error) { +func (e *Env) patchAtenetEgressInject(raw []byte, providers []config.CredentialProvider) ([]byte, error) { var flagsBlock string - if provider.Enabled() { - flagsBlock = emitEgressInjectFlags(provider.Name, provider.Address, provider.ServerName()) + if len(providers) > 0 { + flagsBlock = emitEgressInjectFlags(providers) } return replaceManifestMarker(raw, "#ATE_EGRESS_INJECT_FLAGS", flagsBlock) } // The HTTP and HTTPS listeners share the egress policy and its providers. -func patchAgentgatewayEgressInject(raw []byte, provider config.CredentialProvider) ([]byte, error) { +func patchAgentgatewayEgressInject(raw []byte, providers []config.CredentialProvider) ([]byte, error) { var block string - if provider.Enabled() { - block = fmt.Sprintf(`credentialProviders: + for i, p := range providers { + if i == 0 { + block = "credentialProviders:" + } + block += fmt.Sprintf(` - uriAuthority: %q target: host: %q @@ -333,7 +384,7 @@ func patchAgentgatewayEgressInject(raw []byte, provider config.CredentialProvide hostname: %q cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem - root: /run/servicedns-ca/trust-bundle.pem`, provider.Name, provider.Address, provider.ServerName()) + root: /run/servicedns-ca/trust-bundle.pem`, p.Name, p.Address, p.ServerName()) } return replaceManifestMarker(raw, "#ATE_AGENTGATEWAY_CREDENTIAL_PROVIDERS", block) } @@ -360,12 +411,18 @@ func replaceManifestMarker(raw []byte, marker, block string) ([]byte, error) { return []byte(strings.Join(out, "\n")), nil } -func emitEgressInjectFlags(name, address, serverName string) string { - return fmt.Sprintf(`- --credential-provider-name=%s +// emitEgressInjectFlags writes the name, address and server name flags of each +// provider in order, then the CA and client certificate every provider shares. +func emitEgressInjectFlags(providers []config.CredentialProvider) string { + var flags string + for _, p := range providers { + flags += fmt.Sprintf(`- --credential-provider-name=%s - --credential-provider-address=%s -- --credential-provider-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem -- --credential-provider-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem -- --credential-provider-server-name=%s`, name, address, serverName) +- --credential-provider-server-name=%s +`, p.Name, p.Address, p.ServerName()) + } + return flags + `- --credential-provider-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem +- --credential-provider-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem` } func (e *Env) patchAtenetEgressManifest() ([]byte, error) { @@ -503,8 +560,8 @@ func emitAdditionalEgressExtprocCluster(address, port, serverName string) string port_value: %s`, additionalEgressExtprocCluster, serverName, serverName, additionalEgressExtprocCluster, address, port) } -func (e *Env) applyAtenetEgress(ctx context.Context, provider config.CredentialProvider) error { - manifests, err := e.renderAtenetEgressManifest(ctx, provider) +func (e *Env) applyAtenetEgress(ctx context.Context, providers []config.CredentialProvider) error { + manifests, err := e.renderAtenetEgressManifest(ctx, providers) if err != nil { return err } @@ -518,7 +575,7 @@ func (e *Env) applyAtenetEgress(ctx context.Context, provider config.CredentialP return err } - if running && (e.Cfg.AdditionalEgressExtprocService != "" || provider.Enabled()) { + if running && (e.Cfg.AdditionalEgressExtprocService != "" || len(providers) > 0) { if err := e.Kube.RolloutRestartDeployment(ctx, e.Namespace(), "atenet-egress", time.Now()); err != nil { return err } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index a626532e64..78766c35b6 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -16,7 +16,6 @@ package steps import ( "context" - "errors" "fmt" "os" "reflect" @@ -58,14 +57,16 @@ func TestPatchAtenetEgressInject(t *testing.T) { t.Fatalf("reading egress manifest: %v", err) } + k8s := config.CredentialProvider{Name: config.K8sCredentialProviderName, Address: config.K8sCredentialProviderAddress} + vault := config.CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:8200"} for _, tc := range []struct { - name string - provider config.CredentialProvider - want []string + name string + providers []config.CredentialProvider + want []string }{ { - name: "kubernetes", - provider: config.CredentialProvider{Name: config.K8sCredentialProviderName, Address: config.K8sCredentialProviderAddress}, + name: "kubernetes", + providers: []config.CredentialProvider{k8s}, want: []string{ "--credential-provider-name=k8s.io", "--credential-provider-address=k8s-credential-provider.ate-system.svc:50051", @@ -75,18 +76,31 @@ func TestPatchAtenetEgressInject(t *testing.T) { }, }, { - name: "another provider", - provider: config.CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:8200"}, + name: "another provider", + providers: []config.CredentialProvider{vault}, + want: []string{ + "--credential-provider-name=vault.example.com", + "--credential-provider-address=vault.ate-system.svc:8200", + "--credential-provider-server-name=vault.ate-system.svc", + }, + }, + { + name: "several providers", + providers: []config.CredentialProvider{k8s, vault}, want: []string{ + "--credential-provider-name=k8s.io", + "--credential-provider-address=k8s-credential-provider.ate-system.svc:50051", + "--credential-provider-server-name=k8s-credential-provider.ate-system.svc", "--credential-provider-name=vault.example.com", "--credential-provider-address=vault.ate-system.svc:8200", "--credential-provider-server-name=vault.ate-system.svc", + "--credential-provider-ca-file=", }, }, {name: "off"}, } { t.Run(tc.name, func(t *testing.T) { - patched, err := env.patchAtenetEgressInject(raw, tc.provider) + patched, err := env.patchAtenetEgressInject(raw, tc.providers) if err != nil { t.Fatalf("patchAtenetEgressInject failed: %v", err) } @@ -95,12 +109,16 @@ func TestPatchAtenetEgressInject(t *testing.T) { t.Errorf("patched manifest still contains an unreplaced marker: %q", line) } } + // The gateway pairs the repeated flags up by position. + rest := string(patched) for _, want := range tc.want { - if !strings.Contains(string(patched), want) { - t.Errorf("patched manifest is missing spliced flag %q", want) + i := strings.Index(rest, want) + if i < 0 { + t.Fatalf("patched manifest is missing spliced flag %q, or has it out of order", want) } + rest = rest[i+len(want):] } - if !tc.provider.Enabled() && strings.Contains(string(patched), "- --credential-provider-") { + if len(tc.providers) == 0 && strings.Contains(string(patched), "- --credential-provider-") { t.Error("patched manifest carries provider flags with injection off") } assertEgressManifestParses(t, patched) @@ -109,7 +127,7 @@ func TestPatchAtenetEgressInject(t *testing.T) { // The marker is what the splice keys on; a manifest without it is a // broken install, not a silent no-op. - if _, err := env.patchAtenetEgressInject([]byte("kind: ConfigMap\n"), config.CredentialProvider{}); err == nil { + if _, err := env.patchAtenetEgressInject([]byte("kind: ConfigMap\n"), nil); err == nil { t.Error("patchAtenetEgressInject accepted a manifest without the marker") } } @@ -346,10 +364,10 @@ func TestSystemOverlayDefaultIsBase(t *testing.T) { } // pinnedWorkloads returns the Deployment and StatefulSet names in a rendered -// manifest that carry the cordon-control-plane node pinning, mapped to the -// pool each one selects, and every workload name seen. Only the shared pool -// spreads its replicas, and no pool requires anti-affinity: that would need a -// node per pod. +// manifest that carry the cordon-system-infra-components node pinning, mapped +// to the pool each one selects, and every workload name seen. Only the shared +// pool spreads its replicas, and no pool requires anti-affinity: that would +// need a node per pod. func pinnedWorkloads(t *testing.T, manifest []byte) (pinned map[string]string, all []string) { t.Helper() pinned = map[string]string{} @@ -393,12 +411,12 @@ func pinnedWorkloads(t *testing.T, manifest []byte) (pinned map[string]string, a if podSpec.Affinity["podAntiAffinity"] != nil { t.Errorf("workload %s has pod anti-affinity", obj.Metadata.Name) } - if spread := len(podSpec.TopologySpreadConstraints) > 0; spread != (pool == "ate-control-plane") { + if spread := len(podSpec.TopologySpreadConstraints) > 0; spread != (pool == "ate-system-infra") { t.Errorf("workload %s in pool %s: topology spread = %v", obj.Metadata.Name, pool, spread) } // The spread narrows on the app label; without it the workload would // spread against the whole pool instead of its own replicas. - if pool == "ate-control-plane" && obj.Spec.Template.Metadata.Labels["app"] == "" { + if pool == "ate-system-infra" && obj.Spec.Template.Metadata.Labels["app"] == "" { t.Errorf("workload %s has no app label to spread its replicas by", obj.Metadata.Name) } pinned[obj.Metadata.Name] = pool @@ -406,12 +424,13 @@ func pinnedWorkloads(t *testing.T, manifest []byte) (pinned map[string]string, a return pinned, all } -// Under --cordon-control-plane every control plane apply path has to carry the -// pinning, since each workload reaches the cluster through a different one: -// the system bundle, the lone redeploy files, the podcert overlay, the -// postgres file, the egress variants, and the bundled credential provider. postgres gets a pool of its own; -// every other control plane workload shares one. -func TestRenderCordonControlPlane(t *testing.T) { +// Under --cordon-system-infra-components every system infra apply path has +// to carry the pinning, since each workload reaches the cluster through a +// different one: the system bundle, the lone redeploy files, the podcert +// overlay, the postgres file, the egress variants, and the bundled credential +// provider. postgres gets a pool of its own; every other system infra +// workload shares one. +func TestRenderCordonSystemInfraComponents(t *testing.T) { root := repoRoot(t) for _, tc := range []struct { name string @@ -489,7 +508,7 @@ func TestRenderCordonControlPlane(t *testing.T) { t.Run(tc.name, func(t *testing.T) { cfg := tc.cfg cfg.Root = root - cfg.CordonControlPlane = true + cfg.CordonSystemInfraComponents = true e := &Env{Cfg: &cfg} rendered, err := e.render(tc.path(e)) @@ -501,7 +520,7 @@ func TestRenderCordonControlPlane(t *testing.T) { if !slices.Contains(all, name) { t.Errorf("rendered manifest has no workload %s (found %v)", name, all) } - want := "ate-control-plane" + want := "ate-system-infra" if name == "postgres" { want = "ate-postgres" } @@ -510,10 +529,10 @@ func TestRenderCordonControlPlane(t *testing.T) { } } // The DaemonSet-shaped and demo workloads stay off the pools; only - // the named control plane workloads are pinned. + // the named system infra workloads are pinned. for name := range pinned { if !slices.Contains(tc.want, name) { - t.Errorf("workload %s is pinned but is not a control plane workload", name) + t.Errorf("workload %s is pinned but is not a system infra workload", name) } } }) @@ -522,10 +541,10 @@ func TestRenderCordonControlPlane(t *testing.T) { // The extproc-patched egress manifest arrives as bytes, and the pinning has to // reach it too. -func TestRenderBytesCordonControlPlane(t *testing.T) { +func TestRenderBytesCordonSystemInfraComponents(t *testing.T) { e := &Env{Cfg: &config.Config{ Root: repoRoot(t), - CordonControlPlane: true, + CordonSystemInfraComponents: true, AdditionalEgressExtprocService: "ate-system/foo:50051", }} patched, err := e.patchAtenetEgressManifest() @@ -537,7 +556,7 @@ func TestRenderBytesCordonControlPlane(t *testing.T) { t.Fatalf("renderBytes: %v", err) } pinned, _ := pinnedWorkloads(t, rendered) - if pinned["atenet-egress"] != "ate-control-plane" { + if pinned["atenet-egress"] != "ate-system-infra" { t.Errorf("atenet-egress is not pinned in the composed extproc manifest (pinned: %v)", pinned) } if !strings.Contains(string(rendered), additionalEgressExtprocCluster) { @@ -553,14 +572,14 @@ func TestRenderWithoutCordonLeavesManifestsAlone(t *testing.T) { t.Fatalf("render: %v", err) } if pinned, _ := pinnedWorkloads(t, rendered); len(pinned) != 0 { - t.Errorf("render without --cordon-control-plane pinned %v", pinned) + t.Errorf("render without --cordon-system-infra-components pinned %v", pinned) } raw, err := os.ReadFile(e.Cfg.Manifest("ate-api-server.yaml")) if err != nil { t.Fatal(err) } if string(rendered) != string(raw) { - t.Error("render of a plain file without --cordon-control-plane is not the file's bytes") + t.Error("render of a plain file without --cordon-system-infra-components is not the file's bytes") } } @@ -652,7 +671,7 @@ func TestRenderFakeDataPlane(t *testing.T) { }, { name: "controller file under cordon", - cfg: config.Config{CordonControlPlane: true}, + cfg: config.Config{CordonSystemInfraComponents: true}, path: func(e *Env) string { return e.Cfg.Manifest("ate-controller.yaml") }, wantController: true, }, @@ -756,11 +775,11 @@ func TestRenderFakeDataPlane(t *testing.T) { // Both components compose: the cordon pinning survives the fake // one, and reaches fake-workersync, which stands in for // ate-controller. - if cfg.CordonControlPlane { + if cfg.CordonSystemInfraComponents { pinned, _ := pinnedWorkloads(t, rendered) for _, name := range []string{"ate-controller", fakeWorkersyncDeployment} { - if pinned[name] != "ate-control-plane" { - t.Errorf("under --cordon-control-plane %s is pinned to %q, want ate-control-plane", name, pinned[name]) + if pinned[name] != "ate-system-infra" { + t.Errorf("under --cordon-system-infra-components %s is pinned to %q, want ate-system-infra", name, pinned[name]) } } } @@ -768,8 +787,125 @@ func TestRenderFakeDataPlane(t *testing.T) { } } +// Under --e2e-tuning every path that applies atelet appends the e2e tuning +// flags without dropping the overlay's own args, and a non-e2e kind render +// keeps atelet on its default snapshot upload delay. +func TestRenderE2ETuning(t *testing.T) { + root := repoRoot(t) + const ateletDS = "atelet" + for _, tc := range []struct { + name string + cfg config.Config + path func(e *Env) string + wantAtelet bool + wantArgs []string + }{ + { + name: "base bundle", + cfg: config.Config{Router: config.RouterEnvoy}, + path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, + wantAtelet: true, + wantArgs: []string{"--grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem", "--snapshot-upload-delay=2s"}, + }, + { + name: "kind bundle", + cfg: config.Config{Router: config.RouterEnvoy, Kind: true}, + path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, + wantAtelet: true, + wantArgs: []string{"--localhost-registry-replacement=kind-registry:5000", "--log-level=debug", "--snapshot-upload-delay=2s"}, + }, + { + name: "agentgateway bundle", + cfg: config.Config{Router: config.RouterAgentgateway}, + path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, + wantAtelet: true, + wantArgs: []string{"--grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem", "--snapshot-upload-delay=2s"}, + }, + { + name: "kind agentgateway bundle", + cfg: config.Config{Router: config.RouterAgentgateway, Kind: true}, + path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, + wantAtelet: true, + wantArgs: []string{"--localhost-registry-replacement=kind-registry:5000", "--log-level=debug", "--snapshot-upload-delay=2s"}, + }, + { + name: "atelet file", + path: func(e *Env) string { return e.Cfg.Manifest("atelet.yaml") }, + wantAtelet: true, + wantArgs: []string{"--grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem", "--snapshot-upload-delay=2s"}, + }, + { + name: "kind atelet overlay", + cfg: config.Config{Kind: true}, + path: func(e *Env) string { return e.ateletManifestPath() }, + wantAtelet: true, + wantArgs: []string{"--localhost-registry-replacement=kind-registry:5000", "--log-level=debug", "--snapshot-upload-delay=2s"}, + }, + { + name: "api server file without atelet", + path: func(e *Env) string { return e.Cfg.Manifest("ate-api-server.yaml") }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := tc.cfg + cfg.Root = root + cfg.E2ETuning = true + e := &Env{Cfg: &cfg} + + rendered, err := e.render(tc.path(e)) + if err != nil { + t.Fatalf("render: %v", err) + } + daemonSets, _ := dataPlaneWorkloads(t, rendered) + ds, ok := daemonSets[ateletDS] + if ok != tc.wantAtelet { + t.Fatalf("DaemonSet %s present = %v, want %v", ateletDS, ok, tc.wantAtelet) + } + if !ok { + return + } + args := ds.Spec.Template.Spec.Containers[0].Args + for _, want := range tc.wantArgs { + if !slices.Contains(args, want) { + t.Errorf("atelet args = %v, missing %q", args, want) + } + } + }) + } + + t.Run("kind without e2e-tuning omits snapshot-upload-delay", func(t *testing.T) { + e := &Env{Cfg: &config.Config{Root: root, Kind: true}} + rendered, err := e.render(e.ateletManifestPath()) + if err != nil { + t.Fatalf("render: %v", err) + } + daemonSets, _ := dataPlaneWorkloads(t, rendered) + args := daemonSets[ateletDS].Spec.Template.Spec.Containers[0].Args + if !slices.Contains(args, "--log-level=debug") { + t.Errorf("kind atelet args = %v, missing --log-level=debug", args) + } + for _, arg := range args { + if strings.HasPrefix(arg, "--snapshot-upload-delay") { + t.Errorf("kind atelet without --e2e-tuning carries %q in args %v", arg, args) + } + } + }) + + t.Run("composes with fake-data-plane", func(t *testing.T) { + e := &Env{Cfg: &config.Config{Root: root, E2ETuning: true, FakeDataPlane: true}} + rendered, err := e.render(e.ateletManifestPath()) + if err != nil { + t.Fatalf("render: %v", err) + } + daemonSets, _ := dataPlaneWorkloads(t, rendered) + if args := daemonSets[ateletDS].Spec.Template.Spec.Containers[0].Args; len(args) != 0 { + t.Errorf("fake-atelet args = %v, want none", args) + } + }) +} + // Both paths that deliver the Envoy router pass it the install's allowlist, -// whatever the overlay, and refuse to render without one. +// whatever the overlay, and pass none when the install sets none. func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { const a, b = "spiffe://cluster.local/ns/a/sa/a", "spiffe://cluster.local/ns/b/sa/b" src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} @@ -820,16 +956,193 @@ func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { t.Run("without IDs", func(t *testing.T) { cfg := cfg cfg.IngressAllowedSPIFFEIDs = nil - _, err := tc.render(&Env{Cfg: &cfg, resolver: resolver}) - var required *config.RequiredError - if !errors.As(err, &required) { - t.Fatalf("render error = %v, want a RequiredError", err) + rendered, err := tc.render(&Env{Cfg: &cfg, resolver: resolver}) + if err != nil { + t.Fatalf("render: %v", err) + } + for _, arg := range routerArgs(t, rendered) { + if strings.HasPrefix(arg, "--ingress-allowed-spiffe-ids") { + t.Errorf("atenet-router args include %q, want no allowlist", arg) + } + } + }) + }) + } +} + +func TestRenderAgentgatewayStaticMTLS(t *testing.T) { + const ( + idA = "spiffe://cluster.local/ns/a/sa/a" + idB = "spiffe://cluster.local/ns/b/sa/b" + ) + src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} + resolver := images.NewPrebuilt(src, func(context.Context, string) (string, error) { + return "sha256:" + strings.Repeat("2", 64), nil + }) + for _, tc := range []struct { + name string + render func(*Env) ([]byte, error) + }{ + { + name: "system bundle", + render: func(e *Env) ([]byte, error) { + return e.renderSystemManifests(t.Context()) + }, + }, + { + name: "router redeploy", + render: func(e *Env) ([]byte, error) { + return e.renderAtenetRouterManifest(t.Context()) + }, + }, + { + name: "kind system bundle", + render: func(e *Env) ([]byte, error) { + e.Cfg.Kind = true + return e.renderSystemManifests(t.Context()) + }, + }, + { + name: "egress redeploy", + render: func(e *Env) ([]byte, error) { + return e.renderAtenetEgressManifest(t.Context(), nil) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := &config.Config{ + Root: repoRoot(t), + Router: config.RouterAgentgateway, + IngressAllowedSPIFFEIDs: []string{idA, idB}, + Images: src, + } + e := &Env{Cfg: cfg, resolver: resolver} + manifest, err := tc.render(e) + if err != nil { + t.Fatalf("rendering static-mtls AgentGateway manifests: %v", err) + } + cm := agentgatewayRouterConfig(t, manifest) + for _, headerMap := range []string{"source.connectHeaders", "request.headers"} { + if !strings.Contains(cm.Data["config.yaml"], "size("+headerMap+".raw()['ate-target-actor']) != 1") { + t.Errorf("router config does not reject duplicate actor headers in %s", headerMap) + } + } + var gateway map[string]interface{} + if err := yaml.Unmarshal([]byte(cm.Data["config.yaml"]), &gateway); err != nil { + t.Fatalf("parsing rendered AgentGateway config: %v", err) + } + gateways, ok := gateway["gateways"].(map[string]interface{}) + if !ok { + t.Fatal("rendered AgentGateway config has no gateways mapping") + } + if _, exists := gateways["http"]; exists { + t.Error("static-mtls config still has a plaintext HTTP gateway") + } + https, ok := gateways["https"].(map[string]interface{}) + if !ok { + t.Fatal("static-mtls config has no HTTPS gateway") + } + assertAgentgatewayClientRoot(t, https) + policies := gateway["frontendPolicies"].(map[string]interface{}) + authorization := policies["networkAuthorization"].(map[string]interface{}) + wantRules := []interface{}{ + map[string]interface{}{"allow": fmt.Sprintf("source.spiffeId in [%q,%q]", idA, idB)}, + } + if got := authorization["rules"]; !reflect.DeepEqual(got, wantRules) { + t.Errorf("network authorization rules = %v, want %v", got, wantRules) + } + + binds, ok := gateway["binds"].([]interface{}) + if !ok { + t.Fatal("rendered AgentGateway config has no binds list") + } + var connectTLSListener map[string]interface{} + for _, item := range binds { + bind, ok := item.(map[string]interface{}) + if !ok { + t.Fatalf("invalid bind in rendered AgentGateway config: %T", item) + } + port, ok := bind["port"].(float64) + if !ok { + if _, exists := bind["port"]; exists { + t.Fatalf("invalid bind port in rendered AgentGateway config: %v", bind["port"]) + } + continue + } + if port == 8081 { + t.Error("static-mtls config still has a plaintext CONNECT bind") + } + if port != 8444 { + continue + } + listeners, ok := bind["listeners"].([]interface{}) + if !ok { + t.Fatal("TLS CONNECT bind has no listeners") + } + for _, listenerValue := range listeners { + listener, ok := listenerValue.(map[string]interface{}) + if ok && listener["protocol"] == "HTTPS" { + connectTLSListener = listener + } + } + } + if connectTLSListener == nil { + t.Fatal("static-mtls config has no TLS CONNECT listener") + } + assertAgentgatewayClientRoot(t, connectTLSListener) + + t.Run("without IDs", func(t *testing.T) { + cfg.IngressAllowedSPIFFEIDs = nil + manifest, err := tc.render(e) + if err != nil { + t.Fatalf("rendering static-mtls AgentGateway manifests: %v", err) + } + var gateway map[string]interface{} + if err := yaml.Unmarshal([]byte(agentgatewayRouterConfig(t, manifest).Data["config.yaml"]), &gateway); err != nil { + t.Fatalf("parsing rendered AgentGateway config: %v", err) + } + authorization := gateway["frontendPolicies"].(map[string]interface{})["networkAuthorization"].(map[string]interface{}) + wantRules := []interface{}{map[string]interface{}{"allow": "source.spiffeId in []"}} + if got := authorization["rules"]; !reflect.DeepEqual(got, wantRules) { + t.Errorf("network authorization rules = %v, want %v", got, wantRules) } }) }) } } +func agentgatewayRouterConfig(t *testing.T, manifest []byte) *corev1.ConfigMap { + t.Helper() + for _, doc := range strings.Split(string(manifest), "\n---\n") { + var cm corev1.ConfigMap + if err := yaml.Unmarshal([]byte(doc), &cm); err != nil { + t.Fatalf("rendered document is not valid YAML: %v", err) + } + if cm.Kind == "ConfigMap" && cm.Name == "atenet-router-agentgateway-config" { + if cm.Data["config.yaml"] == "" { + t.Fatal("AgentGateway router ConfigMap has no config.yaml") + } + return &cm + } + } + t.Fatal("rendered manifest has no AgentGateway router ConfigMap") + return nil +} + +func assertAgentgatewayClientRoot(t *testing.T, config map[string]interface{}) { + t.Helper() + tls, ok := config["tls"].(map[string]interface{}) + if !ok { + t.Fatal("TLS listener has no TLS config") + } + if got := tls["root"]; got != "/run/podidentity.podcert.ate.dev/trust-bundle.pem" { + t.Errorf("TLS client CA = %v, want podidentity trust bundle", got) + } + if _, exists := tls["allowedClientSpiffeIds"]; exists { + t.Error("allowlist must use the existing network authorization policy") + } +} + // routerArgs returns the atenet-router container's args from a rendered // manifest. func routerArgs(t *testing.T, manifest []byte) []string { @@ -942,23 +1255,31 @@ func TestAgentgatewayEgressOverlay(t *testing.T) { } func TestRenderAgentgatewayCredentialProvider(t *testing.T) { + k8s := config.CredentialProvider{Name: config.K8sCredentialProviderName, Address: config.K8sCredentialProviderAddress} + vault := config.CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:8200"} for _, tc := range []struct { - name string - provider config.CredentialProvider + name string + providers []config.CredentialProvider }{ {name: "disabled"}, - {name: "kubernetes", provider: config.CredentialProvider{Name: config.K8sCredentialProviderName, Address: config.K8sCredentialProviderAddress}}, - {name: "custom", provider: config.CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:8200"}}, + {name: "kubernetes", providers: []config.CredentialProvider{k8s}}, + {name: "custom", providers: []config.CredentialProvider{vault}}, + {name: "several providers", providers: []config.CredentialProvider{k8s, vault}}, } { t.Run(tc.name, func(t *testing.T) { src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} e := &Env{ - Cfg: &config.Config{Root: repoRoot(t), Router: config.RouterAgentgateway, Images: src}, + Cfg: &config.Config{ + Root: repoRoot(t), + Router: config.RouterAgentgateway, + Images: src, + IngressAllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/client/sa/client"}, + }, resolver: images.NewPrebuilt(src, func(context.Context, string) (string, error) { return "sha256:" + strings.Repeat("2", 64), nil }), } - built, err := e.renderAtenetEgressManifest(t.Context(), tc.provider) + built, err := e.renderAtenetEgressManifest(t.Context(), tc.providers) if err != nil { t.Fatal(err) } @@ -1002,24 +1323,27 @@ func TestRenderAgentgatewayCredentialProvider(t *testing.T) { if err := policy.YNode().Decode(&got); err != nil { t.Fatal(err) } - if !tc.provider.Enabled() { + if len(tc.providers) == 0 { if len(got.Providers) != 0 { t.Errorf("%s unexpectedly has credential providers: %v", protocol, got.Providers) } continue } - want := []map[string]any{{ - "uriAuthority": tc.provider.Name, - "target": map[string]any{ - "host": tc.provider.Address, - "policies": map[string]any{"backendTLS": map[string]any{ - "hostname": tc.provider.ServerName(), - "cert": "/run/podidentity.podcert.ate.dev/credential-bundle.pem", - "key": "/run/podidentity.podcert.ate.dev/credential-bundle.pem", - "root": "/run/servicedns-ca/trust-bundle.pem", - }}, - }, - }} + var want []map[string]any + for _, p := range tc.providers { + want = append(want, map[string]any{ + "uriAuthority": p.Name, + "target": map[string]any{ + "host": p.Address, + "policies": map[string]any{"backendTLS": map[string]any{ + "hostname": p.ServerName(), + "cert": "/run/podidentity.podcert.ate.dev/credential-bundle.pem", + "key": "/run/podidentity.podcert.ate.dev/credential-bundle.pem", + "root": "/run/servicedns-ca/trust-bundle.pem", + }}, + }, + }) + } if !reflect.DeepEqual(got.Providers, want) { t.Errorf("credential providers = %v, want %v", got.Providers, want) } @@ -1034,7 +1358,7 @@ func TestRenderAgentgatewayCredentialProvider(t *testing.T) { func TestPatchAgentgatewayCredentialProviderRequiresOneMarker(t *testing.T) { for _, raw := range []string{"", "#ATE_AGENTGATEWAY_CREDENTIAL_PROVIDERS\n#ATE_AGENTGATEWAY_CREDENTIAL_PROVIDERS\n"} { - if _, err := patchAgentgatewayEgressInject([]byte(raw), config.CredentialProvider{}); err == nil { + if _, err := patchAgentgatewayEgressInject([]byte(raw), nil); err == nil { t.Errorf("accepted a manifest without exactly one marker: %q", raw) } } @@ -1135,8 +1459,7 @@ func TestApplyOtelEndpointOverride(t *testing.T) { // A pre-built install renders the envoy router, egress, and system manifests // without building anything: envoy-dataplane is pinned from the image // repository like every ko image, so neither docker nor KO_DOCKER_REPO is -// needed. Releases do not publish it, so this holds for a repository someone -// published it to. +// needed. func TestRenderAtenetEgressManifestPrebuilt(t *testing.T) { const digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222" src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} @@ -1157,7 +1480,7 @@ func TestRenderAtenetEgressManifestPrebuilt(t *testing.T) { }{ { name: "egress", - render: func() ([]byte, error) { return e.renderAtenetEgressManifest(t.Context(), config.CredentialProvider{}) }, + render: func() ([]byte, error) { return e.renderAtenetEgressManifest(t.Context(), nil) }, }, { name: "router", @@ -1188,9 +1511,8 @@ func TestRenderAtenetEgressManifestPrebuilt(t *testing.T) { } } -// An install from release images that needs envoy-dataplane fails with a -// message saying releases do not publish it and how to proceed, rather than a -// bare registry error. +// An install from an image repository without a readable envoy-dataplane fails +// with a message saying how to proceed, rather than a bare registry error. func TestDockerfileImagePrebuiltNotPublished(t *testing.T) { src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} e := &Env{ @@ -1200,12 +1522,12 @@ func TestDockerfileImagePrebuiltNotPublished(t *testing.T) { }), } - _, err := e.dockerfileImage(t.Context(), envoyDataplaneImage, envoyDataplaneDockefile) + _, err := e.dockerfileImage(t.Context(), images.EnvoyDataplane) if err == nil { t.Fatal("dockerfileImage() error = nil, want one") } for _, want := range []string{ - "envoy-dataplane is built from a Dockerfile, and releases do not publish it", + "envoy-dataplane is built from a Dockerfile, and the image repository has no readable copy", "--atenet-dataplane=agentgateway", "resolving example.com/substrate/envoy-dataplane:v1.2.3 to a digest: MANIFEST_UNKNOWN", } { @@ -1254,7 +1576,7 @@ func TestRenderAtenetEgressManifestEnvoyConcurrency(t *testing.T) { return digest, nil }), } - out, err := e.renderAtenetEgressManifest(t.Context(), config.CredentialProvider{}) + out, err := e.renderAtenetEgressManifest(t.Context(), nil) if err != nil { t.Fatalf("renderAtenetEgressManifest() error = %v", err) } @@ -1276,7 +1598,7 @@ func TestRenderAtenetEgressManifestEnvoyConcurrency(t *testing.T) { return digest, nil }), } - out, err := e.renderAtenetEgressManifest(t.Context(), config.CredentialProvider{}) + out, err := e.renderAtenetEgressManifest(t.Context(), nil) if err != nil { t.Fatalf("renderAtenetEgressManifest() error = %v", err) } diff --git a/cmd/ate-setup/internal/steps/postgres.go b/cmd/ate-setup/internal/steps/postgres.go index 3e8be19b4a..d3544adf7f 100644 --- a/cmd/ate-setup/internal/steps/postgres.go +++ b/cmd/ate-setup/internal/steps/postgres.go @@ -90,21 +90,21 @@ func (e *Env) waitAndSetupBundledPostgres(ctx context.Context) error { } // The size10 PostgreSQL container. Deliberately no CPU limit: under -// --cordon-control-plane the dedicated ate-postgres pool keeps the pod alone -// on its node, so a limit would only add CFS throttling on checkpoint and -// autovacuum bursts. Without that flag the pod shares whatever node fits the -// request, and the missing limit lets it contend with its neighbors. Memory -// request == limit keeps eviction ordering equivalent to a Guaranteed pod, -// which matters because postgres cannot release shared_buffers under pressure. -// postgres-size10/postgres-config-patch.yaml is tuned to these numbers, so they -// move together. +// --cordon-system-infra-components the dedicated ate-postgres pool keeps the +// pod alone on its node, so a limit would only add CFS throttling on +// checkpoint and autovacuum bursts. Without that flag the pod shares whatever +// node fits the request, and the missing limit lets it contend with its +// neighbors. Memory request == limit keeps eviction ordering equivalent to a +// Guaranteed pod, which matters because postgres cannot release +// shared_buffers under pressure. postgres-size10/postgres-config-patch.yaml +// is tuned to these numbers, so they move together. const ( size10PostgresCPURequest = "80" size10PostgresMemory = "140Gi" ) -// postgresPoolSelector selects the nodes of the pool the cordon-control-plane -// component pins the bundled PostgreSQL to. +// postgresPoolSelector selects the nodes of the pool the +// cordon-system-infra-components component pins the bundled PostgreSQL to. const postgresPoolSelector = "ate.dev/workloadType=ate-postgres" // postgresPlan says where ateapi's store comes from: the bundled StatefulSet, @@ -242,11 +242,12 @@ func setPostgresStorageClass(objs []*unstructured.Unstructured, class string) er } // requirePostgresPool refuses to apply the bundled PostgreSQL under -// --cordon-control-plane when no node carries the ate-postgres label. The -// StatefulSet deletes its running pod before creating the replacement, so -// without the pool the database goes down and the new pod stays Pending. +// --cordon-system-infra-components when no node carries the ate-postgres +// label. The StatefulSet deletes its running pod before creating the +// replacement, so without the pool the database goes down and the new pod +// stays Pending. func (e *Env) requirePostgresPool(ctx context.Context) error { - if !e.Cfg.CordonControlPlane { + if !e.Cfg.CordonSystemInfraComponents { return nil } nodes, err := e.Kube.Typed.CoreV1().Nodes().List(ctx, metav1.ListOptions{ @@ -257,9 +258,9 @@ func (e *Env) requirePostgresPool(ctx context.Context) error { return fmt.Errorf("while listing the %s nodes: %w", postgresPoolSelector, err) } if len(nodes.Items) == 0 { - return fmt.Errorf("--cordon-control-plane pins the bundled PostgreSQL to nodes labeled %s, and the cluster has none: "+ + return fmt.Errorf("--cordon-system-infra-components pins the bundled PostgreSQL to nodes labeled %s, and the cluster has none: "+ "create a one-node pool labeled and tainted %s:NoSchedule in the zone of the postgres volume first "+ - "(see docs/upgrade.md#cordoned-control-plane-postgres-pool)", postgresPoolSelector, postgresPoolSelector) + "(see docs/architecture.md#system-infra-isolation)", postgresPoolSelector, postgresPoolSelector) } return nil } diff --git a/cmd/ate-setup/internal/steps/postgres_test.go b/cmd/ate-setup/internal/steps/postgres_test.go index a9bfdc1cd3..715c6e68c7 100644 --- a/cmd/ate-setup/internal/steps/postgres_test.go +++ b/cmd/ate-setup/internal/steps/postgres_test.go @@ -315,11 +315,11 @@ func labeledNode(name string, labels map[string]string) runtime.Object { return &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels}} } -// Under --cordon-control-plane the StatefulSet's rolling update deletes the -// running postgres before its replacement schedules, so a cluster with no -// ate-postgres node must be refused before anything is applied. +// Under --cordon-system-infra-components the StatefulSet's rolling update +// deletes the running postgres before its replacement schedules, so a cluster +// with no ate-postgres node must be refused before anything is applied. func TestRequirePostgresPool(t *testing.T) { - controlPlane := labeledNode("cp-1", map[string]string{"ate.dev/workloadType": "ate-control-plane"}) + systemInfra := labeledNode("infra-1", map[string]string{"ate.dev/workloadType": "ate-system-infra"}) postgres := labeledNode("pg-1", map[string]string{"ate.dev/workloadType": "ate-postgres"}) for _, tc := range []struct { name string @@ -327,22 +327,22 @@ func TestRequirePostgresPool(t *testing.T) { nodes []runtime.Object wantErr bool }{ - {name: "not cordoned", nodes: []runtime.Object{controlPlane}}, - {name: "cordoned with a postgres node", cordon: true, nodes: []runtime.Object{controlPlane, postgres}}, - {name: "cordoned without a postgres node", cordon: true, nodes: []runtime.Object{controlPlane}, wantErr: true}, + {name: "not cordoned", nodes: []runtime.Object{systemInfra}}, + {name: "cordoned with a postgres node", cordon: true, nodes: []runtime.Object{systemInfra, postgres}}, + {name: "cordoned without a postgres node", cordon: true, nodes: []runtime.Object{systemInfra}, wantErr: true}, {name: "cordoned with no nodes", cordon: true, wantErr: true}, } { t.Run(tc.name, func(t *testing.T) { e := &Env{ - Cfg: &config.Config{CordonControlPlane: tc.cordon}, + Cfg: &config.Config{CordonSystemInfraComponents: tc.cordon}, Kube: fakeKube(t, tc.nodes...), } err := e.requirePostgresPool(t.Context()) if gotErr := err != nil; gotErr != tc.wantErr { t.Fatalf("requirePostgresPool() error = %v, wantErr %v", err, tc.wantErr) } - if err != nil && !strings.Contains(err.Error(), "docs/upgrade.md") { - t.Errorf("requirePostgresPool() error = %q, want it to point at docs/upgrade.md", err) + if err != nil && !strings.Contains(err.Error(), "docs/architecture.md") { + t.Errorf("requirePostgresPool() error = %q, want it to point at docs/architecture.md", err) } }) } @@ -352,7 +352,7 @@ func TestRequirePostgresPool(t *testing.T) { // take to the StatefulSet, so the pool check has to sit in front of it. func TestApplyPostgresRequiresPostgresPool(t *testing.T) { e := &Env{ - Cfg: &config.Config{Root: repoRoot(t), CordonControlPlane: true}, + Cfg: &config.Config{Root: repoRoot(t), CordonSystemInfraComponents: true}, Kube: fakeKube(t), } err := e.applyPostgres(t.Context()) diff --git a/cmd/ate-setup/internal/steps/publish.go b/cmd/ate-setup/internal/steps/publish.go index cf36277fae..f98326e733 100644 --- a/cmd/ate-setup/internal/steps/publish.go +++ b/cmd/ate-setup/internal/steps/publish.go @@ -168,22 +168,24 @@ type PublishReleaseOptions struct { } // releaseImageRefs returns the tagged reference of every image a release -// publishes to repo: one per images.ReleaseComponents entry. +// publishes to repo: one per images.ReleaseComponents and +// images.ReleaseDockerfileImages entry. func releaseImageRefs(repo, tag string) []string { repo = strings.TrimSuffix(repo, "/") - pkgs := images.ReleaseComponents() - refs := make([]string, 0, len(pkgs)) - for _, pkg := range pkgs { + var refs []string + for _, pkg := range images.ReleaseComponents() { refs = append(refs, repo+"/"+images.ImageName(pkg)+":"+tag) } + for _, img := range images.ReleaseDockerfileImages() { + refs = append(refs, repo+"/"+img.Name+":"+tag) + } return refs } // PublishReleaseImages builds and pushes every image a release publishes, the -// ko images in images.ReleaseComponents, all tagged with the build version, -// and writes their pushed references to w. The version has to be a release -// tag. Releases do not publish envoy-dataplane, the one Dockerfile-built -// image, so nothing here needs docker. +// ko images in images.ReleaseComponents and the images.ReleaseDockerfileImages +// built with docker buildx, all tagged with the build version, and writes +// their pushed references to w. The version has to be a release tag. // // If the registry already holds every image for the tag on every platform, // nothing is built unless opts.Overwrite is set, so rerunning a release never @@ -234,11 +236,29 @@ func (e *Env) PublishReleaseImages(ctx context.Context, w io.Writer, opts Publis labels := []string{ images.SourceLabel + "=" + images.SourceURL, images.RevisionLabel + "=" + revision, + images.VersionLabel + "=" + tag, + } + // The Dockerfile images are slower to build and likelier to fail, so they + // go first: a failure there publishes nothing under the tag. They ignore + // DOCKER_BUILD_FLAGS, so a release restores no build cache. + var dockerFlags []string + for _, label := range labels { + dockerFlags = append(dockerFlags, "--label="+label) + } + var refs []string + for _, img := range images.ReleaseDockerfileImages() { + ref, err := images.PublishDockerfileImage(ctx, e.Cfg.Root, e.Cfg.KODockerRepo, img.Name, tag, + e.Cfg.Path(img.Context), e.Cfg.KODefaultPlatforms, dockerFlags) + if err != nil { + return err + } + refs = append(refs, ref) } - refs, err := runner.BuildTagged(ctx, tag, labels, pkgs...) + koRefs, err := runner.BuildTagged(ctx, tag, labels, pkgs...) if err != nil { return err } + refs = append(refs, koRefs...) incomplete, err := images.CheckPublished(ctx, tagged, platforms) if err != nil { diff --git a/cmd/ate-setup/internal/steps/publish_test.go b/cmd/ate-setup/internal/steps/publish_test.go index 90288247a9..686af7f105 100644 --- a/cmd/ate-setup/internal/steps/publish_test.go +++ b/cmd/ate-setup/internal/steps/publish_test.go @@ -120,23 +120,26 @@ func TestPublishReleaseImagesRejectsANonReleaseTag(t *testing.T) { } } -// The release covers every released component, each under the name and tag -// that `deploy --image-repo --image-tag` looks up, and nothing releases leave -// out. +// The release covers every released component and envoy-dataplane, each under +// the name and tag that `deploy --image-repo --image-tag` looks up, and nothing +// releases leave out. func TestReleaseImageRefs(t *testing.T) { refs := releaseImageRefs("example.com/substrate/", "v1") - if want := len(images.ReleaseComponents()); len(refs) != want { + if want := len(images.ReleaseComponents()) + len(images.ReleaseDockerfileImages()); len(refs) != want { t.Fatalf("releaseImageRefs() returned %d refs, want %d", len(refs), want) } - for _, want := range []string{"example.com/substrate/ateapi:v1", "example.com/substrate/objectstore-plugin:v1"} { + for _, want := range []string{ + "example.com/substrate/ateapi:v1", + "example.com/substrate/envoy-dataplane:v1", + "example.com/substrate/kubernetes-secrets:v1", + "example.com/substrate/objectstore-plugin:v1", + } { if !slices.Contains(refs, want) { t.Errorf("releaseImageRefs() = %v, want %s", refs, want) } } for _, unwanted := range []string{ "example.com/substrate/counter:v1", - "example.com/substrate/kubernetes-secrets:v1", - "example.com/substrate/envoy-dataplane:v1", } { if slices.Contains(refs, unwanted) { t.Errorf("releaseImageRefs() = %v, want no %s", refs, unwanted) diff --git a/cmd/ate-setup/internal/steps/substrate_test.go b/cmd/ate-setup/internal/steps/substrate_test.go index d29c0ec174..0f748c8daf 100644 --- a/cmd/ate-setup/internal/steps/substrate_test.go +++ b/cmd/ate-setup/internal/steps/substrate_test.go @@ -40,8 +40,7 @@ func TestActorTemplateFromManifestDemos(t *testing.T) { t.Fatalf("RepoRoot: %v", err) } - // The values benchmarking/workloads/deploy.sh substitutes, beyond the - // ${BUCKET_NAME} every installer provides. + // The values benchmarking/workloads/deploy.sh substitutes. benchmarkValues := map[string]string{ "${ACTOR_CPU_MILLICORES}": "1000", "${ACTOR_MEMORY_BYTES}": "268435456", @@ -113,7 +112,7 @@ func TestActorTemplateFromManifestDemos(t *testing.T) { if err != nil { t.Fatalf("ReadFile: %v", err) } - manifest := strings.ReplaceAll(string(data), "${BUCKET_NAME}", "ate-snapshots") + manifest := string(data) for placeholder, value := range tc.values { manifest = strings.ReplaceAll(manifest, placeholder, value) } @@ -127,6 +126,10 @@ func TestActorTemplateFromManifestDemos(t *testing.T) { if got := resources.ActorTemplateRefFromActorTemplate(template); got != tc.want { t.Errorf("manifest names template %s, want %s", got, tc.want) } + // The API rejects storageLocation without its preview gate. + if got := template.GetSnapshotConfig().GetStorageLocation(); got != "" { + t.Errorf("manifest sets storageLocation %q, want none", got) + } }) } } diff --git a/cmd/ate-setup/main.go b/cmd/ate-setup/main.go index d115f75c5c..2f8c227b3d 100644 --- a/cmd/ate-setup/main.go +++ b/cmd/ate-setup/main.go @@ -13,10 +13,7 @@ // limitations under the License. // Command ate-setup installs and tears down Agent Substrate on a Kubernetes -// cluster. It is the installer; hack/install-ate.sh is a shim that translates -// its historical flags onto these commands. See cmd/ate-setup/commands.md for -// the flag-by-flag mapping and cmd/ate-setup/cli-diff.md for what the -// translation does not cover. +// cluster. See cmd/ate-setup/commands.md for its commands. package main import ( diff --git a/cmd/ate-setup/preflight_test.go b/cmd/ate-setup/preflight_test.go index 729907dc9e..8ac37e7f4e 100644 --- a/cmd/ate-setup/preflight_test.go +++ b/cmd/ate-setup/preflight_test.go @@ -40,9 +40,8 @@ func repoRoot(t *testing.T) string { return strings.TrimSpace(string(out)) } -// Each invocation gets its own command tree and flags, including invocations -// made by the real shell shims. The test executable runs the installer without -// rebuilding it or contacting a real cluster. +// Each invocation gets its own command tree and flags. The test executable runs +// the installer without rebuilding it or contacting a real cluster. func TestInstallerPreflightProcess(t *testing.T) { if os.Getenv("ATE_SETUP_PREFLIGHT_TEST") != "1" { return @@ -66,17 +65,6 @@ func runPreflightInstaller(t *testing.T, kubeconfig, dockerStatus string, args . } for name, source := range map[string]string{ "docker": "#!/bin/sh\necho called >> \"$PREFLIGHT_DOCKER_LOG\"\nexit \"$PREFLIGHT_DOCKER_STATUS\"\n", - "go": `#!/bin/sh -if [ "$*" = "env GOARCH" ]; then - echo amd64 -elif [ "$1" = run ] && [ "$2" = ./cmd/ate-setup ]; then - shift 2 - exec "$PREFLIGHT_EXECUTABLE" -test.run='^TestInstallerPreflightProcess$' -- "$@" -else - echo "unexpected go invocation: $*" >&2 - exit 1 -fi -`, } { if err := os.WriteFile(filepath.Join(bin, name), []byte(source), 0o755); err != nil { t.Fatal(err) @@ -93,7 +81,6 @@ fi "NO_DEV_ENV=true", "ATE_NO_REPORT_DEFAULTS=1", "ATE_SETUP_PREFLIGHT_TEST=1", - "PREFLIGHT_EXECUTABLE=" + executable, "PREFLIGHT_DOCKER_STATUS=" + dockerStatus, "PREFLIGHT_DOCKER_LOG=" + filepath.Join(bin, "docker.log"), } @@ -197,6 +184,21 @@ func TestKindInstallerHelpNeedsNoCluster(t *testing.T) { } } +// Cobra rejects an unknown command only at the root, so a typo one level down +// used to print help and exit 0 having installed nothing. +func TestInstallerRejectsUnknownSubcommand(t *testing.T) { + for _, args := range [][]string{ + {"deploy", "demo-counter"}, + {"deploy", "demo", "demo-counter"}, + } { + out, code := runPreflightInstaller(t, preflightKubeconfig(t, ""), "0", args...) + want := `unknown command "demo-counter"` + if code == 0 || !strings.Contains(out, want) { + t.Errorf("%v: exit=%d, want an error containing %q\n%s", args, code, want, out) + } + } +} + func TestInstallerControlPlaneCheckPreservesErrorsAndAllowsInstalledSystem(t *testing.T) { for _, installed := range []bool{false, true} { t.Run(fmt.Sprintf("installed=%t", installed), func(t *testing.T) { diff --git a/cmd/ateapi/internal/apivalidation/actor_template.go b/cmd/ateapi/internal/apivalidation/actor_template.go index cad1b5aa34..492af02aae 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template.go +++ b/cmd/ateapi/internal/apivalidation/actor_template.go @@ -16,10 +16,8 @@ package apivalidation import ( "context" - "fmt" "go.ate.dev/substrate/api/ateapi/v1" - "go.ate.dev/substrate/internal/preview" "go.ate.dev/substrate/internal/resources" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/util/sets" @@ -166,19 +164,9 @@ func ValidateCustom_Container_Image(_ context.Context, _ operation.Operation, fl } // ValidateCustom_SnapshotConfig_StorageLocation ensures an -// ActorTemplate's snapshotConfig.location is a well-formed +// ActorTemplate's snapshotConfig.storageLocation is a well-formed // URI with a bucket, so a bad location fails fast. -// -// The field is ignored while the PerTemplateSnapshotLocation gate is off, so a -// declarative ifDisabled=forbidden would reject the templates that set it -// on servers without the gate. Instead, with the gate on, a request that -// did not opt in to preview features is forbidden from setting it. -func ValidateCustom_SnapshotConfig_StorageLocation(ctx context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { - if preview.IsEnabled(preview.GatePerTemplateSnapshotLocation) && preview.IsOptedOut(ctx) { - return field.ErrorList{field.Forbidden(fldPath, fmt.Sprintf( - "requires a client that opts in to preview features (kubectl ate --preview, or the %s header) while the %s preview gate is enabled", - preview.Header, preview.GatePerTemplateSnapshotLocation))} - } +func ValidateCustom_SnapshotConfig_StorageLocation(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { if err := resources.ValidateSnapshotLocation(*value); err != nil { return field.ErrorList{field.Invalid(fldPath, *value, err.Error())} } diff --git a/cmd/ateapi/internal/apivalidation/actor_template_test.go b/cmd/ateapi/internal/apivalidation/actor_template_test.go index a2d8de5e56..4c111dc51b 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_template_test.go @@ -167,9 +167,9 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) { })}, nil, }, { - "snapshot_config.storage_location is not needed when the preview gate is off", + "valid storage_location", &ateapi.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapi.ActorTemplate) { - tmpl.SnapshotConfig.StorageLocation = "" + tmpl.SnapshotConfig.StorageLocation = "gs://my-bucket/snapshots" })}, nil, }, { @@ -230,6 +230,10 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) { want = field.ErrorList{field.Forbidden(field.NewPath("actor_template", "golden_egress_policy"), "")} wantWithPreview = map[string]field.ErrorList{"GoldenEgressPolicy": tt.want, "ALL": tt.want} } + if tt.req.GetActorTemplate().GetSnapshotConfig().GetStorageLocation() != "" { + want = field.ErrorList{field.Forbidden(field.NewPath("actor_template", "snapshot_config", "storage_location"), "")} + wantWithPreview = map[string]field.ErrorList{"PerTemplateSnapshotLocation": tt.want, "ALL": tt.want} + } assertValidate(t, func() field.ErrorList { return ValidateCreateActorTemplateRequest(context.Background(), tt.req) }, want, wantWithPreview) @@ -241,6 +245,7 @@ func TestValidateCreateActorTemplateRequest_PreviewRestricted(t *testing.T) { setPreviewForTest(t, "ALL") req := &ateapi.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapi.ActorTemplate) { tmpl.Volumes = []*ateapi.Volume{{Name: "data", External: &ateapi.ExternalVolumeSource{Template: &ateapi.ExternalVolumeTemplate{CapacityBytes: 10 << 30, StorageClassName: "fast-ssd"}}}} + tmpl.SnapshotConfig.StorageLocation = "gs://my-bucket/snapshots" })} t.Run("allowed", func(t *testing.T) { @@ -256,9 +261,9 @@ func TestValidateCreateActorTemplateRequest_PreviewRestricted(t *testing.T) { }) } -func TestValidateCreateActorTemplateRequest_PerTemplateSnapshotLocationOptIn(t *testing.T) { +func TestValidateCreateActorTemplateRequest_PerTemplateSnapshotLocation(t *testing.T) { storageLocation := field.NewPath("actor_template", "snapshot_config", "storage_location") - withoutLocation := func(tmpl *ateapi.ActorTemplate) { tmpl.SnapshotConfig.StorageLocation = "" } + withLocation := func(tmpl *ateapi.ActorTemplate) { tmpl.SnapshotConfig.StorageLocation = "gs://my-bucket/snapshots" } tests := []struct { name string gates string @@ -266,29 +271,34 @@ func TestValidateCreateActorTemplateRequest_PerTemplateSnapshotLocationOptIn(t * mutate func(*ateapi.ActorTemplate) want field.ErrorList }{{ + name: "gate off, opted in, location set", + mutate: withLocation, + want: field.ErrorList{field.Forbidden(storageLocation, "")}, + }, { name: "gate off, opted out, location set", optOut: true, + mutate: withLocation, + want: field.ErrorList{field.Forbidden(storageLocation, "")}, }, { name: "gate off, opted out, no location", optOut: true, - mutate: withoutLocation, }, { name: "gate on, opted out, location set", gates: "PerTemplateSnapshotLocation", optOut: true, + mutate: withLocation, want: field.ErrorList{field.Forbidden(storageLocation, "")}, }, { name: "gate on, opted out, no location", gates: "PerTemplateSnapshotLocation", optOut: true, - mutate: withoutLocation, }, { - name: "gate on, opted in, location set", - gates: "PerTemplateSnapshotLocation", - }, { - name: "gate on, opted in, no location", + name: "gate on, opted in, location set", gates: "PerTemplateSnapshotLocation", - mutate: withoutLocation, + mutate: withLocation, + }, { + name: "gate on, opted in, no location", + gates: "PerTemplateSnapshotLocation", }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -471,13 +481,15 @@ func TestValidateActorTemplate(t *testing.T) { mutate: func(tmpl *ateapi.ActorTemplate) { tmpl.SnapshotConfig.StorageLocation = "gs://" + strings.Repeat("x", 1020) }, - want: field.ErrorList{field.TooLong(field.NewPath("snapshot_config", "storage_location"), nil, 1024).WithOrigin("maxLength")}, + want: field.ErrorList{field.Forbidden(field.NewPath("snapshot_config", "storage_location"), "")}, + wantWithPreview: map[string]field.ErrorList{"PerTemplateSnapshotLocation": {field.TooLong(field.NewPath("snapshot_config", "storage_location"), nil, 1024).WithOrigin("maxLength")}}, }, { name: "missing storage_location", mutate: func(tmpl *ateapi.ActorTemplate) { tmpl.SnapshotConfig.StorageLocation = "" }, // Optional with or without the gate: an empty field falls back to // --snapshot-location. - want: nil, + want: nil, + wantWithPreview: map[string]field.ErrorList{"PerTemplateSnapshotLocation": nil}, }, { name: "unspecified snapshot scope", mutate: func(tmpl *ateapi.ActorTemplate) { @@ -962,7 +974,6 @@ func validActorTemplate(mutations ...func(*ateapi.ActorTemplate)) *ateapi.ActorT Metadata: &ateapi.ResourceMetadata{Atespace: "ns1", Name: "tmpl-a"}, Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: "gs://my-bucket/snapshots", PreferredFidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, diff --git a/cmd/ateapi/internal/apivalidation/validation_test.go b/cmd/ateapi/internal/apivalidation/validation_test.go index 43b315af58..768d7e1cde 100644 --- a/cmd/ateapi/internal/apivalidation/validation_test.go +++ b/cmd/ateapi/internal/apivalidation/validation_test.go @@ -1068,7 +1068,7 @@ func validObjectSnapshot(mutate ...func(*ateapi.ObjectSnapshot)) *ateapi.ObjectS func validSnapshotStorage(mutate ...func(*ateapi.SnapshotStorage)) *ateapi.SnapshotStorage { ss := &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: validObjectSnapshot(), } for _, m := range mutate { @@ -1098,7 +1098,7 @@ func badSnapshot(mutate ...func(*ateapi.Snapshot)) *ateapi.Snapshot { breakIt := func(s *ateapi.Snapshot) { s.Fidelity = ateapi.SnapshotFidelity(4) s.DurableSnapshot = &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: ""}, } } @@ -1113,7 +1113,7 @@ func TestValidateSnapshot(t *testing.T) { fidelityPath := field.NewPath("fidelity") localityPath := field.NewPath("locality") durableSnapshotPath := field.NewPath("durable_snapshot") - statusPath := durableSnapshotPath.Child("status") + statePath := durableSnapshotPath.Child("state") objectPath := durableSnapshotPath.Child("object") uriPath := objectPath.Child("snapshot_uri") @@ -1185,25 +1185,25 @@ func TestValidateSnapshot(t *testing.T) { obj: valid(func(s *ateapi.Snapshot) { s.DurableSnapshot = nil }), }, { - name: "missing durable_snapshot.status", + name: "missing durable_snapshot.state", obj: valid(func(s *ateapi.Snapshot) { - s.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED + s.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED }), - want: field.ErrorList{field.Required(statusPath, "")}, + want: field.ErrorList{field.Required(statePath, "")}, }, { - name: "negative durable_snapshot.status", + name: "negative durable_snapshot.state", obj: valid(func(s *ateapi.Snapshot) { - s.DurableSnapshot.Status = ateapi.SnapshotStorageStatus(-1) + s.DurableSnapshot.State = ateapi.SnapshotStorageState(-1) }), - want: field.ErrorList{field.Invalid(statusPath, nil, "").WithOrigin("minimum")}, + want: field.ErrorList{field.Invalid(statePath, nil, "").WithOrigin("minimum")}, }, { - name: "durable_snapshot.status above the enum", + name: "durable_snapshot.state above the enum", obj: valid(func(s *ateapi.Snapshot) { - s.DurableSnapshot.Status = ateapi.SnapshotStorageStatus(3) + s.DurableSnapshot.State = ateapi.SnapshotStorageState(3) }), - want: field.ErrorList{field.Invalid(statusPath, nil, "").WithOrigin("maximum")}, + want: field.ErrorList{field.Invalid(statePath, nil, "").WithOrigin("maximum")}, }, { name: "missing durable_snapshot.object", diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index 5d32a05e7c..bfe1672f01 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -7682,6 +7682,83 @@ func Validate_RevertActorRequest( return errs } +// Validate_RuntimeOverhead validates an instance of RuntimeOverhead according +// to declarative validation rules in the API schema. +func Validate_RuntimeOverhead( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *v1.RuntimeOverhead) (errs field.ErrorList) { + + { // field v1.RuntimeOverhead.MemoryBytes + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 1099511627776); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *v1.RuntimeOverhead) *int64 { + return &oldObj.MemoryBytes + }) + errs = append(errs, fn(fldPath.Child("memory_bytes"), &obj.MemoryBytes, oldVal, oldObj != nil)...) + } + + { // field v1.RuntimeOverhead.MemoryBytesPerGib + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 1073741824); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *v1.RuntimeOverhead) *int64 { + return &oldObj.MemoryBytesPerGib + }) + errs = append(errs, fn(fldPath.Child("memory_bytes_per_gib"), &obj.MemoryBytesPerGib, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_SecurityContext validates an instance of SecurityContext according // to declarative validation rules in the API schema. func Validate_SecurityContext( @@ -8208,6 +8285,13 @@ func Validate_SnapshotConfig( } // call field-attached validations earlyReturn := false + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "PerTemplateSnapshotLocation", false, validate.ForbiddenValue).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "PerTemplateSnapshotLocation", false, validate.OptionalValue).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } @@ -8499,10 +8583,10 @@ func Validate_SnapshotStorage( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *v1.SnapshotStorage) (errs field.ErrorList) { - { // field v1.SnapshotStorage.Status + { // field v1.SnapshotStorage.State fn := func( fldPath *field.Path, - obj, oldObj *v1.SnapshotStorageStatus, + obj, oldObj *v1.SnapshotStorageState, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -8528,10 +8612,10 @@ func Validate_SnapshotStorage( return } oldVal := safe.Field(oldObj, - func(oldObj *v1.SnapshotStorage) *v1.SnapshotStorageStatus { - return &oldObj.Status + func(oldObj *v1.SnapshotStorage) *v1.SnapshotStorageState { + return &oldObj.State }) - errs = append(errs, fn(fldPath.Child("status"), &obj.Status, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("state"), &obj.State, oldVal, oldObj != nil)...) } { // field v1.SnapshotStorage.Object @@ -11091,6 +11175,36 @@ func Validate_WorkerSandboxRuntime( errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) } + { // field v1.WorkerSandboxRuntime.ActorOverhead + fn := func( + fldPath *field.Path, + obj, oldObj *v1.RuntimeOverhead, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_RuntimeOverhead(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *v1.WorkerSandboxRuntime) *v1.RuntimeOverhead { + return oldObj.ActorOverhead + }) + errs = append(errs, fn(fldPath.Child("actor_overhead"), obj.ActorOverhead, oldVal, oldObj != nil)...) + } + return errs } diff --git a/cmd/ateapi/internal/authz/authz_util.go b/cmd/ateapi/internal/authz/authz_util.go index a85729f4d7..1e1734d6cd 100644 --- a/cmd/ateapi/internal/authz/authz_util.go +++ b/cmd/ateapi/internal/authz/authz_util.go @@ -233,7 +233,7 @@ func EnsureStoreAndModel(ctx context.Context, pool *pgxpool.Pool, fgaServer *ser // every check, independent of any stored AccessPolicy. type SystemPrincipals struct { ControllerSPIFFEID string - EgressGatewaySPIFFEID string + EgressGatewaySPIFFEIDs []string IngressRouterSPIFFEID string AteletSPIFFEID string APIServerSidecarSPIFFEID string @@ -289,16 +289,22 @@ func parseBootstrapOwners(ids []string) (map[string]struct{}, error) { // parseSystemPrincipals validates system's SPIFFE IDs and maps each OpenFGA // user string to the system component roles it holds. func parseSystemPrincipals(system SystemPrincipals) (map[string][]string, error) { - bindings := []struct { + if len(system.EgressGatewaySPIFFEIDs) == 0 { + return nil, fmt.Errorf("no %s SPIFFE ID", RoleEgressGateway) + } + type binding struct { role string spiffeID string - }{ + } + bindings := []binding{ {role: RoleController, spiffeID: system.ControllerSPIFFEID}, - {role: RoleEgressGateway, spiffeID: system.EgressGatewaySPIFFEID}, {role: RoleIngressRouter, spiffeID: system.IngressRouterSPIFFEID}, {role: RoleAtelet, spiffeID: system.AteletSPIFFEID}, {role: RoleAPIServerSidecar, spiffeID: system.APIServerSidecarSPIFFEID}, } + for _, id := range system.EgressGatewaySPIFFEIDs { + bindings = append(bindings, binding{role: RoleEgressGateway, spiffeID: id}) + } out := make(map[string][]string, len(bindings)) for _, b := range bindings { if _, err := spiffeid.FromString(b.spiffeID); err != nil { diff --git a/cmd/ateapi/internal/authz/authz_util_test.go b/cmd/ateapi/internal/authz/authz_util_test.go index 77a13c26cf..da08a0887f 100644 --- a/cmd/ateapi/internal/authz/authz_util_test.go +++ b/cmd/ateapi/internal/authz/authz_util_test.go @@ -26,7 +26,7 @@ import ( func testSystemPrincipals() SystemPrincipals { return SystemPrincipals{ ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEIDs: []string{installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace)}, IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), @@ -96,7 +96,7 @@ func TestParseSystemPrincipals(t *testing.T) { const router = "spiffe://cluster.local/ns/ate-system/sa/atenet-router" got, err := parseSystemPrincipals(SystemPrincipals{ ControllerSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/ate-controller", - EgressGatewaySPIFFEID: router, + EgressGatewaySPIFFEIDs: []string{router, "spiffe://cluster.local/ns/plugins/sa/token-exchange"}, IngressRouterSPIFFEID: router, AteletSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/atelet", APIServerSidecarSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/ate-api-server", @@ -106,7 +106,8 @@ func TestParseSystemPrincipals(t *testing.T) { } want := map[string][]string{ "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-controller": {RoleController}, - "user:spiffe%3A//cluster.local/ns/ate-system/sa/atenet-router": {RoleEgressGateway, RoleIngressRouter}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/atenet-router": {RoleIngressRouter, RoleEgressGateway}, + "user:spiffe%3A//cluster.local/ns/plugins/sa/token-exchange": {RoleEgressGateway}, "user:spiffe%3A//cluster.local/ns/ate-system/sa/atelet": {RoleAtelet}, "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-api-server": {RoleAPIServerSidecar}, } @@ -123,13 +124,16 @@ func TestNewRejectsInvalidSystemPrincipals(t *testing.T) { wantErr string }{ {"empty controller SPIFFE ID", func(p *SystemPrincipals) { p.ControllerSPIFFEID = "" }, "invalid controller SPIFFE ID"}, - {"empty egress gateway SPIFFE ID", func(p *SystemPrincipals) { p.EgressGatewaySPIFFEID = "" }, "invalid egress_gateway SPIFFE ID"}, + {"no egress gateway SPIFFE IDs", func(p *SystemPrincipals) { p.EgressGatewaySPIFFEIDs = nil }, "no egress_gateway SPIFFE ID"}, + {"empty egress gateway SPIFFE ID in list", func(p *SystemPrincipals) { + p.EgressGatewaySPIFFEIDs = []string{defaults.EgressGatewaySPIFFEIDs[0], ""} + }, "invalid egress_gateway SPIFFE ID"}, {"empty ingress router SPIFFE ID", func(p *SystemPrincipals) { p.IngressRouterSPIFFEID = "" }, "invalid ingress_router SPIFFE ID"}, {"empty atelet SPIFFE ID", func(p *SystemPrincipals) { p.AteletSPIFFEID = "" }, "invalid atelet SPIFFE ID"}, {"empty api server sidecar SPIFFE ID", func(p *SystemPrincipals) { p.APIServerSidecarSPIFFEID = "" }, "invalid api_server_sidecar SPIFFE ID"}, {"bare ServiceAccount name", func(p *SystemPrincipals) { p.ControllerSPIFFEID = "ate-controller" }, "invalid controller SPIFFE ID"}, {"non-spiffe URI", func(p *SystemPrincipals) { - p.EgressGatewaySPIFFEID = "https://cluster.local/ns/ate-system/sa/atenet-egress" + p.EgressGatewaySPIFFEIDs = []string{"https://cluster.local/ns/ate-system/sa/atenet-egress"} }, "invalid egress_gateway SPIFFE ID"}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/authz/interceptor.go b/cmd/ateapi/internal/authz/interceptor.go index 586c16f35e..e950bf68c8 100644 --- a/cmd/ateapi/internal/authz/interceptor.go +++ b/cmd/ateapi/internal/authz/interceptor.go @@ -20,6 +20,8 @@ import ( "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/principal" "google.golang.org/grpc" + "google.golang.org/grpc/reflection/grpc_reflection_v1" + "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" ) // UnaryServerInterceptor returns a gRPC unary interceptor that enforces per-RPC @@ -56,3 +58,24 @@ func UnaryServerInterceptor(authorizer *Authorizer) grpc.UnaryServerInterceptor return handler(ctx, req) } } + +// streamRules lists the streaming RPCs ate-api-server serves; any other stream +// is denied. ateapi.proto defines none (see +// TestDefaultRPCPermissions_CoversEveryServedRPC). gRPC reflection is open to +// any authenticated caller, like Kubernetes API discovery: it returns only the +// served API schema, which is public in api/ateapi/v1/ateapi.proto. +var streamRules = map[string]bool{ + grpc_reflection_v1.ServerReflection_ServerReflectionInfo_FullMethodName: true, + grpc_reflection_v1alpha.ServerReflection_ServerReflectionInfo_FullMethodName: true, +} + +// StreamServerInterceptor returns a gRPC stream interceptor that denies every +// streaming RPC not in streamRules. +func StreamServerInterceptor() grpc.StreamServerInterceptor { + return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + if !streamRules[info.FullMethod] { + return apierror.PermissionDenied("permission denied: no authorization rule for %s", info.FullMethod) + } + return handler(srv, ss) + } +} diff --git a/cmd/ateapi/internal/authz/interceptor_test.go b/cmd/ateapi/internal/authz/interceptor_test.go index 33b140000b..b20bd6a7a6 100644 --- a/cmd/ateapi/internal/authz/interceptor_test.go +++ b/cmd/ateapi/internal/authz/interceptor_test.go @@ -23,6 +23,8 @@ import ( "go.ate.dev/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/reflection/grpc_reflection_v1" + "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" ) func setupTestAuthorizer(t *testing.T) *Authorizer { @@ -235,6 +237,37 @@ func TestUnaryServerInterceptor_UnregisteredAndHandlerAuthorizedRPCs(t *testing. } } +// Reflection streams pass through; any other stream is denied before its +// handler runs. +func TestStreamServerInterceptor(t *testing.T) { + tests := []struct { + name string + fullMethod string + wantCode codes.Code + }{ + {"reflection v1 passes", grpc_reflection_v1.ServerReflection_ServerReflectionInfo_FullMethodName, codes.OK}, + {"reflection v1alpha passes", grpc_reflection_v1alpha.ServerReflection_ServerReflectionInfo_FullMethodName, codes.OK}, + {"unregistered stream is denied", "/substrate.ateapi.v1.Control/NotARealMethod", codes.PermissionDenied}, + {"unary RPC called as a stream is denied", ateapi.Control_ListAtespaces_FullMethodName, codes.PermissionDenied}, + {"handler-authorized RPC called as a stream is denied", ateapi.WorkerService_RegisterWorker_FullMethodName, codes.PermissionDenied}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + handlerCalled := false + err := StreamServerInterceptor()(nil, nil, &grpc.StreamServerInfo{FullMethod: tc.fullMethod}, func(any, grpc.ServerStream) error { + handlerCalled = true + return nil + }) + if apierror.Code(err) != tc.wantCode { + t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", apierror.Code(err), tc.wantCode, err) + } + if wantHandler := tc.wantCode == codes.OK; handlerCalled != wantHandler { + t.Fatalf("handlerCalled = %v, want %v", handlerCalled, wantHandler) + } + }) + } +} + func TestUnaryServerInterceptor_ActorAndTemplateChecks(t *testing.T) { ctx := context.Background() pool := startPostgres(t) @@ -593,7 +626,7 @@ func TestUnaryServerInterceptor_SystemPrincipals(t *testing.T) { ) authorizer, _, err := New(ctx, pool, fgaServer, nil, SystemPrincipals{ ControllerSPIFFEID: controllerID, - EgressGatewaySPIFFEID: egressID, + EgressGatewaySPIFFEIDs: []string{egressID}, IngressRouterSPIFFEID: routerID, AteletSPIFFEID: ateletID, APIServerSidecarSPIFFEID: pluginID, diff --git a/cmd/ateapi/internal/authz/registry_test.go b/cmd/ateapi/internal/authz/registry_test.go index 53487c2797..57d2b6bc77 100644 --- a/cmd/ateapi/internal/authz/registry_test.go +++ b/cmd/ateapi/internal/authz/registry_test.go @@ -692,9 +692,9 @@ func TestDefaultRPCPermissions_CoversEveryServedRPC(t *testing.T) { m := methods.Get(j) fullMethod := "/" + string(sd.FullName()) + "/" + string(m.Name()) if m.IsStreamingClient() || m.IsStreamingServer() { - // The authz interceptor is unary-only, so a streaming RPC would go - // unchecked. - t.Errorf("%s is a streaming RPC, which the authz interceptor does not check", fullMethod) + // StreamServerInterceptor denies any stream not in streamRules, + // and has no per-request checks like defaultRPCPermissions. + t.Errorf("%s is a streaming RPC, which StreamServerInterceptor denies", fullMethod) } served[fullMethod] = true } diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index ffee8080ea..bd092788e0 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -191,10 +191,13 @@ func (s *ServiceImpl) resolveTagSource(ctx context.Context, actorAtespace string default: return nil, apierror.FailedPrecondition("source Tag has an invalid scope") } - // TODO: Refuse a tag with delete_time set. DeleteTag may already have - // collected its snapshot. + // DeleteTag may already have collected the snapshot of a tag with + // delete_time set. + if tag.GetMetadata().GetDeleteTime() != nil { + return nil, apierror.FailedPrecondition("source Tag %s/%s is being deleted", tagRef.GetAtespace(), tagRef.GetName()) + } // A tag might have an empty Snapshot URI if the tag creation failed or is ongoing. - if tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { + if tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) == "" { return nil, apierror.FailedPrecondition("source Tag is still being created or failed creation") } // TODO: Permit compatible VOLUMES snapshots when runtimes can extract portable data. @@ -264,6 +267,9 @@ func (s *RPCService) UpdateActor(ctx context.Context, req *ateapi.UpdateActorReq defer lease.Close() storedActor, err := s.impl.UpdateActor(ctx, actorRef, store.PreconditionFrom(inActor), func(toUpdate *ateapi.Actor) error { + if toUpdate.GetMetadata().GetDeleteTime() != nil { + return apierror.FailedPrecondition("Actor %s is being deleted", actorRef) + } // Status and Metadata are server-owned fields. status, metadata := toUpdate.GetStatus(), toUpdate.GetMetadata() // Whole-object replace: clear first, so a field the client left unset is diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index b68f8ff73b..d346b43146 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -42,7 +42,6 @@ func validActorTemplate(mutations ...func(*ateapi.ActorTemplate)) *ateapi.ActorT Metadata: &ateapi.ResourceMetadata{Atespace: "ns1", Name: "tmpl-a"}, Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: "gs://my-bucket/snapshots", PreferredFidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, @@ -57,7 +56,9 @@ func validActorTemplate(mutations ...func(*ateapi.ActorTemplate)) *ateapi.ActorT // while the atespace is missing, and succeeds once the atespace exists. func TestCreateActorTemplate(t *testing.T) { persistence := newTestPersistence(t) - s := &RPCService{impl: newServiceImpl(persistence, nil)} + impl := newServiceImpl(persistence, nil) + impl.clusterSnapshotLocation = "gs://cluster-bucket/" + s := &RPCService{impl: impl} ctx := context.Background() req := func(atespace, name string) *ateapi.CreateActorTemplateRequest { return &ateapi.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapi.ActorTemplate) { @@ -83,7 +84,7 @@ func TestCreateActorTemplate(t *testing.T) { // TestCreateActorTemplate_RequiresSnapshotLocation pins that create rejects, // with FailedPrecondition, a template whose actors would have no snapshot -// location: neither an honored storageLocation nor --snapshot-location. +// location: neither storageLocation nor --snapshot-location. func TestCreateActorTemplate_RequiresSnapshotLocation(t *testing.T) { tests := []struct { name string @@ -95,11 +96,9 @@ func TestCreateActorTemplate_RequiresSnapshotLocation(t *testing.T) { }{{ name: "gate off, cluster location set", clusterSnapshotLocation: "gs://cluster-bucket/", - location: "gs://my-bucket/snapshots", wantCode: codes.OK, }, { name: "gate off, no cluster location", - location: "gs://my-bucket/snapshots", wantCode: codes.FailedPrecondition, }, { name: "gate on, location set, no cluster location", @@ -153,7 +152,9 @@ func TestCreateActorTemplate_RequiresSnapshotLocation(t *testing.T) { // the only guard. func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) { persistence := newTestPersistence(t) - s := &RPCService{impl: newServiceImpl(persistence, nil)} + impl := newServiceImpl(persistence, nil) + impl.clusterSnapshotLocation = "gs://cluster-bucket/" + s := &RPCService{impl: impl} ctx := context.Background() if _, err := persistence.CreateAtespace(ctx, &ateapi.Atespace{Metadata: &ateapi.ResourceMetadata{Name: "ns1"}}); err != nil { @@ -259,7 +260,7 @@ func TestDeleteActorTemplate(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), actorURI.Name(), actorURI.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), actorURI.Name(), actorURI.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } }) var tag *ateapi.Tag diff --git a/cmd/ateapi/internal/controlapi/actor_test.go b/cmd/ateapi/internal/controlapi/actor_test.go index 29b4057422..f1c3b5fd96 100644 --- a/cmd/ateapi/internal/controlapi/actor_test.go +++ b/cmd/ateapi/internal/controlapi/actor_test.go @@ -405,7 +405,7 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { snap := newPausedSnapshot(t, 1, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "pause-snap", "node-1") if name == "uploaded-paused-actor" { snap.DurableSnapshot = &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: uploaded.String()}, } } @@ -615,9 +615,9 @@ func TestUpdateActor_RepointTemplateStorageLocation(t *testing.T) { }) if tt.snapshot != nil { uri := tt.snapshot(t, actor) - storageStatus := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + storageState := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED if tt.pending { - storageStatus = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + storageState = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING } actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 @@ -625,7 +625,7 @@ func TestUpdateActor_RepointTemplateStorageLocation(t *testing.T) { s.LastAssignedGeneration = tt.lastAssignedGen } s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot", uri, storageStatus), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot", uri, storageState), } }) } @@ -990,7 +990,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { Scope: ateapi.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapi.TagStatus{ ActorTemplateUid: tmpl.GetMetadata().GetUid(), - Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), someActorUID, "gs://bucket/atespaces/team-a/tags/"+someActorUID, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), someActorUID, "gs://bucket/atespaces/team-a/tags/"+someActorUID, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), }, } wantCode := codes.OK @@ -1021,7 +1021,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { actor := &ateapi.Actor{Metadata: &ateapi.ResourceMetadata{Atespace: "team-a", Name: "actor"}, ActorTemplate: resources.ActorTemplateRefFromActorTemplate(tmpl).ToObjectRef()} if scenario == "explicit tag" { tag.Metadata.Name = "explicit" - tag.Status.Snapshot = newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "explicit", "gs://bucket/atespaces/team-a/tags/explicit", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + tag.Status.Snapshot = newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "explicit", "gs://bucket/atespaces/team-a/tags/explicit", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if _, err := persistence.CreateTag(ctx, tag); err != nil { t.Fatal(err) } @@ -1035,8 +1035,8 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { if err != nil { return } - gotSnap := findLatestDurableSnapshot(created.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) - if gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) || gotSnap.GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { + gotSnap := findLatestDurableSnapshot(created.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) + if gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) || gotSnap.GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { t.Fatalf("incorrect initial status: %v", created.GetStatus()) } if _, err := persistence.GetEgressPolicy(ctx, resources.ActorRefFromActor(created)); !errors.Is(err, store.ErrNotFound) { @@ -1050,7 +1050,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { if _, err := persistence.UpdateActor(ctx, resources.ActorRefFromActor(created), store.PreconditionFrom(created), func(db *ateapi.Actor) error { db.Status.LastAssignedGeneration = 1 db.Status.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } return nil }); err != nil { diff --git a/cmd/ateapi/internal/controlapi/crash_test.go b/cmd/ateapi/internal/controlapi/crash_test.go index 03d6d925c6..2e9e584585 100644 --- a/cmd/ateapi/internal/controlapi/crash_test.go +++ b/cmd/ateapi/internal/controlapi/crash_test.go @@ -65,7 +65,7 @@ func seedActor(t *testing.T, ctx context.Context, st store.Interface, actorRef r }, LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "reserved-snapshot", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "reserved-snapshot", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, }) @@ -112,7 +112,7 @@ func seedUnboundActor(t *testing.T, ctx context.Context, st store.Interface, act State: ateapi.ActorState_ACTOR_STATE_RUNNING, LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "reserved-snapshot", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "reserved-snapshot", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, }) @@ -130,7 +130,7 @@ func assertCrashed(t *testing.T, ctx context.Context, st store.Interface, actorR t.Errorf("status = %v, want %v", got.GetStatus().GetState(), ateapi.ActorState_ACTOR_STATE_CRASHED) } // Keep the snapshot uri for debugging. - if findLatestDurableSnapshot(got.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) == nil { + if findLatestDurableSnapshot(got.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) == nil { t.Error(`in-progress durable snapshot URI = "", want preserved`) } if got.GetStatus().GetWorkerAssignment() != nil { diff --git a/cmd/ateapi/internal/controlapi/export_test.go b/cmd/ateapi/internal/controlapi/export_test.go index c1953553a8..81f1d80aec 100644 --- a/cmd/ateapi/internal/controlapi/export_test.go +++ b/cmd/ateapi/internal/controlapi/export_test.go @@ -97,7 +97,7 @@ func ReleaseWindowWorkflows() []ReleaseWindowWorkflow { Name: "suspend", State: ateapi.ActorState_ACTOR_STATE_SUSPENDING, Snapshot: func(t *testing.T) *ateapi.Snapshot { - return newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + return newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) }, // As for the async suspend, the finalize commit is the first actor update. Finalize: func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapi.Actor) error { diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go index e589656a95..c8e6edaff2 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go @@ -42,10 +42,9 @@ func TestActorTemplateCRUD(t *testing.T) { created, err := tc.client.CreateActorTemplate(ctx, &ateapi.CreateActorTemplateRequest{ ActorTemplate: &ateapi.ActorTemplate{ - Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a"}, - Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, - SnapshotConfig: &ateapi.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 1<<30)}, + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a"}, + Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, + Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 1<<30)}, // Server-owned status on the request is ignored. Status: &ateapi.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapi.GoldenSnapshotStatus{ @@ -61,7 +60,6 @@ func TestActorTemplateCRUD(t *testing.T) { Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a", Version: 1}, Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: "gs://my-bucket/snapshots", PreferredFidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, GoldenSnapshotConfig: &ateapi.GoldenSnapshotConfig{Mode: ateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_ENABLED}, }, @@ -74,10 +72,9 @@ func TestActorTemplateCRUD(t *testing.T) { _, err = tc.client.CreateActorTemplate(ctx, &ateapi.CreateActorTemplateRequest{ ActorTemplate: &ateapi.ActorTemplate{ - Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a"}, - Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, - SnapshotConfig: &ateapi.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a"}, + Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, + Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, }, }) assertGrpcError(t, err, codes.AlreadyExists, "ActorTemplate "+testAtespace+"/tmpl-a already exists") @@ -236,7 +233,6 @@ func TestActorLifecycleWithoutGoldenSnapshot(t *testing.T) { tmpl, err := tc.client.CreateActorTemplate(ctx, &ateapi.CreateActorTemplateRequest{ActorTemplate: &ateapi.ActorTemplate{ Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "no-golden"}, SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: testStorageLocation, GoldenSnapshotConfig: &ateapi.GoldenSnapshotConfig{Mode: ateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED}, }, Containers: []*ateapi.Container{{Name: "main", Image: "main@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", Command: []string{"/main"}}}, @@ -413,8 +409,7 @@ func TestCreateActorTemplate_ExternalVolumesPreview(t *testing.T) { Name: "data", External: &ateapi.ExternalVolumeSource{Template: &ateapi.ExternalVolumeTemplate{StorageClassName: "standard", CapacityBytes: 1 << 30}}, }}, - Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 1<<30)}, - SnapshotConfig: &ateapi.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, + Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 1<<30)}, }, }) if status.Code(err) != tt.wantCode { @@ -426,9 +421,8 @@ func TestCreateActorTemplate_ExternalVolumesPreview(t *testing.T) { // TestCreateActorTemplate_PerTemplateSnapshotLocationPreview pins how // snapshotConfig.storageLocation combines the PerTemplateSnapshotLocation gate -// with the client's preview opt-in. With the gate off the field is ignored, -// so any client may set it. With the gate on, only a client that opts in may -// set it; a template that leaves it empty uses the server's +// with the client's preview opt-in. Only a client that opts in may set it, and +// only with the gate on. A template that leaves it empty uses the server's // --snapshot-location. func TestCreateActorTemplate_PerTemplateSnapshotLocationPreview(t *testing.T) { ns := namespaceForTest("ns-template-storage-location-preview") @@ -442,8 +436,9 @@ func TestCreateActorTemplate_PerTemplateSnapshotLocationPreview(t *testing.T) { location string wantCode codes.Code }{ - {name: "gate off, no opt-in, location set", optIn: false, location: "gs://my-bucket/snapshots", wantCode: codes.OK}, - {name: "gate off, opt-in, location set", optIn: true, location: "gs://my-bucket/snapshots", wantCode: codes.OK}, + {name: "gate off, no opt-in, location set", optIn: false, location: "gs://my-bucket/snapshots", wantCode: codes.InvalidArgument}, + {name: "gate off, opt-in, location set", optIn: true, location: "gs://my-bucket/snapshots", wantCode: codes.InvalidArgument}, + {name: "gate off, opt-in, no location", optIn: true, wantCode: codes.OK}, {name: "gate on, no opt-in, location set", gates: []string{"PerTemplateSnapshotLocation"}, optIn: false, location: "gs://my-bucket/snapshots", wantCode: codes.InvalidArgument}, {name: "gate on, no opt-in, no location", gates: []string{"PerTemplateSnapshotLocation"}, optIn: false, wantCode: codes.OK}, {name: "gate on, opt-in, location set", gates: []string{"PerTemplateSnapshotLocation"}, optIn: true, location: "gs://my-bucket/snapshots", wantCode: codes.OK}, diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index dae1b1eae2..2dec2eb4a0 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -278,10 +278,9 @@ func TestCreateActor_SubstrateTemplateRef(t *testing.T) { ctx := context.Background() if _, err := tc.client.CreateActorTemplate(ctx, &ateapi.CreateActorTemplateRequest{ ActorTemplate: &ateapi.ActorTemplate{ - Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "sub-tmpl"}, - Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, - SnapshotConfig: &ateapi.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "sub-tmpl"}, + Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, + Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, }, }); err != nil { t.Fatalf("CreateActorTemplate failed: %v", err) @@ -402,8 +401,7 @@ func TestCreateActor_RejectsSnapshotWithExternalVolumes(t *testing.T) { defer tc.cleanup() template, err := tc.client.CreateActorTemplate(context.Background(), &ateapi.CreateActorTemplateRequest{ ActorTemplate: &ateapi.ActorTemplate{ - Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl1"}, - SnapshotConfig: &ateapi.SnapshotConfig{StorageLocation: "gs://snapshots"}, + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl1"}, Containers: []*ateapi.Container{{ Name: "main", Image: "main@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", VolumeMounts: []*ateapi.VolumeMount{{Name: "data", MountPath: "/data"}}, }}, @@ -449,7 +447,7 @@ func TestCreateActor_PendingTag(t *testing.T) { // Simulates a tag creation that failed while writing the snapshot to // external storage. pending := seedTag(t, tc, "pending-source", "pending", func(tag *ateapi.Tag) { - tag.Status.Snapshot.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + tag.Status.Snapshot.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING tag.Status.ActorTemplateUid = tmpl.GetMetadata().GetUid() }) tagRef := &ateapi.ObjectRef{Atespace: testAtespace, Name: "pending"} @@ -467,7 +465,7 @@ func TestCreateActor_PendingTag(t *testing.T) { if _, err := tc.persistence.UpdateTag(ctx, resources.TagRefFromTag(pending), store.PreconditionFrom(pending), func(toUpdate *ateapi.Tag) error { - toUpdate.Status.Snapshot.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + toUpdate.Status.Snapshot.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED return nil }); err != nil { t.Fatalf("finalizing the tag: %v", err) @@ -489,6 +487,60 @@ func TestCreateActor_PendingTag(t *testing.T) { } } +// TestCreateActor_FromDeletingTag verifies an Actor cannot be created from a tag +// with delete_time set, since DeleteTag may already have collected its snapshot. +func TestCreateActor_FromDeletingTag(t *testing.T) { + ns := namespaceForTest("ns-deleting-tag") + tc := setupTest(t, ns) + defer tc.cleanup() + ctx := context.Background() + tmpl := createTemplate(t, tc, ns) + + tag := seedTag(t, tc, "deleting-source", "deleting", func(tag *ateapi.Tag) { + tag.Status.ActorTemplateUid = tmpl.GetMetadata().GetUid() + }) + if _, err := tc.persistence.MarkTagForDeletion(ctx, resources.TagRefFromTag(tag), store.DeletePreconditions{}, nil); err != nil { + t.Fatalf("MarkTagForDeletion: %v", err) + } + + _, err := tc.client.CreateActor(ctx, &ateapi.CreateActorRequest{Actor: &ateapi.Actor{ + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "clone"}, + ActorTemplate: &ateapi.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + SourceTag: resources.TagRefFromTag(tag).ToObjectRef(), + }}) + assertGrpcError(t, err, codes.FailedPrecondition, "source Tag "+testAtespace+"/deleting is being deleted") +} + +// TestCreateActor_FromDeletingGoldenTag verifies an Actor created without a source +// tag is refused while its template's golden tag has delete_time set. +func TestCreateActor_FromDeletingGoldenTag(t *testing.T) { + ns := namespaceForTest("ns-deleting-golden-tag") + tc := setupTest(t, ns) + defer tc.cleanup() + ctx := context.Background() + tmpl := createTemplate(t, tc, ns) + + golden := seedTag(t, tc, "golden-source", "golden", func(tag *ateapi.Tag) { + tag.Status.ActorTemplateUid = tmpl.GetMetadata().GetUid() + }) + goldenRef := resources.TagRefFromTag(golden) + if _, err := tc.persistence.UpdateActorTemplate(ctx, resources.ActorTemplateRefFromActorTemplate(tmpl), store.PreconditionFrom(tmpl), func(toUpdate *ateapi.ActorTemplate) error { + toUpdate.Status = &ateapi.ActorTemplateStatus{GoldenSnapshotStatus: &ateapi.GoldenSnapshotStatus{GoldenTag: goldenRef.ToObjectRef()}} + return nil + }); err != nil { + t.Fatalf("recording the golden tag: %v", err) + } + if _, err := tc.persistence.MarkTagForDeletion(ctx, goldenRef, store.DeletePreconditions{}, nil); err != nil { + t.Fatalf("MarkTagForDeletion: %v", err) + } + + _, err := tc.client.CreateActor(ctx, &ateapi.CreateActorRequest{Actor: &ateapi.Actor{ + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "actor"}, + ActorTemplate: &ateapi.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}) + assertGrpcError(t, err, codes.FailedPrecondition, "source Tag "+testAtespace+"/golden is being deleted") +} + // TestGetActor_Found tests that an existing actor can be retrieved. func TestGetActor_Found(t *testing.T) { ns := namespaceForTest("ns-get-found") @@ -841,9 +893,8 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", VolumeMounts: []*ateapi.VolumeMount{{Name: "data", MountPath: tmpl.mountPath}}, }}, - Volumes: tmpl.volumes, - SnapshotConfig: &ateapi.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, + Volumes: tmpl.volumes, + Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, }, }); err != nil { t.Fatalf("CreateActorTemplate %s failed: %v", name, err) @@ -1059,6 +1110,34 @@ func TestUpdateActor_NotFound(t *testing.T) { assertGrpcError(t, err, codes.NotFound, "actor test-atespace/does-not-exist not found") } +func TestUpdateActor_FailsIfDeleting(t *testing.T) { + ns := namespaceForTest("ns-update-actor-deleting") + tc := setupTest(t, ns) + defer tc.cleanup() + createTemplate(t, tc, ns) + + ctx := context.Background() + actorRef := resources.ActorRef{Atespace: testAtespace, Name: "id1"} + if _, err := tc.persistence.CreateActor(ctx, &ateapi.Actor{ + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + Status: &ateapi.ActorStatus{State: ateapi.ActorState_ACTOR_STATE_DELETING}, + ActorTemplate: &ateapi.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }); err != nil { + t.Fatalf("CreateActor: %v", err) + } + marked, err := tc.persistence.MarkActorForDeletion(ctx, actorRef, store.DeletePreconditions{}, nil) + if err != nil { + t.Fatalf("MarkActorForDeletion: %v", err) + } + + _, err = tc.client.UpdateActor(ctx, &ateapi.UpdateActorRequest{Actor: &ateapi.Actor{ + Metadata: marked.GetMetadata(), + ActorTemplate: marked.GetActorTemplate(), + WorkerSelector: &ateapi.Selector{MatchLabels: map[string]string{"tier": "paid"}}, + }}) + assertGrpcError(t, err, codes.FailedPrecondition, "Actor test-atespace/id1 is being deleted") +} + func TestUpdateActor_StampsFullSpanIdentity(t *testing.T) { ns := namespaceForTest("ns-span-update") tc := setupTest(t, ns) @@ -1198,7 +1277,7 @@ func TestDeleteActor_MissingSnapshotBucket(t *testing.T) { if state != ateapi.ActorState_ACTOR_STATE_SUSPENDED { // A failed upload leaves only the in-progress URI, even // when a prior deletion already marked the actor DELETING. - snap.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + snap.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING } toUpdate.Status.Snapshots = []*ateapi.Snapshot{snap} return nil @@ -3340,7 +3419,6 @@ func createDataFidelityTemplate(t *testing.T, tc *testContext, ns string) *ateap Name: "tmpl1", }, SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: testStorageLocation, PreferredFidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, Containers: []*ateapi.Container{{ @@ -4213,8 +4291,8 @@ func TestResumeActor_SyncSuspendKeepsLocalCopy(t *testing.T) { if snap == nil || snap.GetLocality() != "node1" { t.Fatalf("sync snapshot = %v, want its local copy on node1 recorded", snap) } - if got := snap.GetDurableSnapshot().GetStatus(); got != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { - t.Fatalf("sync snapshot durable status = %v, want COMPLETED", got) + if got := snap.GetDurableSnapshot().GetState(); got != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { + t.Fatalf("sync snapshot durable state = %v, want COMPLETED", got) } if got := snap.GetActorTemplateUid(); got != tmpl.GetMetadata().GetUid() { t.Fatalf("snapshot template UID = %q, want original template %q", got, tmpl.GetMetadata().GetUid()) @@ -4713,8 +4791,8 @@ func TestResumeActor_PausedPendingSnapshotRefusesRuntimeFallback(t *testing.T) { if paused == nil { t.Fatalf("pause left no local snapshot: %v", pauseResp.GetActor().GetStatus()) } - if got := paused.GetDurableSnapshot().GetStatus(); got != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { - t.Fatalf("pause checkpoint durable status = %v, want PENDING (not yet uploaded)", got) + if got := paused.GetDurableSnapshot().GetState(); got != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING { + t.Fatalf("pause checkpoint durable state = %v, want PENDING (not yet uploaded)", got) } durable := durableSnapshot(pauseResp.GetActor().GetStatus()) if durable == nil || durable.GetGeneration() >= paused.GetGeneration() { @@ -4751,7 +4829,7 @@ func TestResumeActor_PausedPendingSnapshotRefusesRuntimeFallback(t *testing.T) { } stillLocal := localSnapshot(after.GetStatus()) if stillLocal == nil || stillLocal.GetUuid() != paused.GetUuid() || - stillLocal.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { + stillLocal.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING { t.Errorf("pause checkpoint after the refused resume = %v, want %v still PENDING", stillLocal, paused) } if got := durableSnapshotURI(after.GetStatus()); got != durable.GetDurableSnapshot().GetObject().GetSnapshotUri() { @@ -5796,7 +5874,7 @@ func TestSuspendActor_SnapshotConflictPreservesState(t *testing.T) { pending = snap } } - if pending.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING || pending.GetLocality() != "" { + if pending.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING || pending.GetLocality() != "" { t.Fatalf("in-progress snapshot = %v, want PENDING without locality", pending) } assertSnapshotCollected(t, tc, pending.GetDurableSnapshot().GetObject().GetSnapshotUri()) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index cb950eb5c7..e628c0145e 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -63,8 +63,8 @@ import ( const ( testAtespace = "test-atespace" testActorID = "id1" - // testStorageLocation is the snapshot_config.storage_location the test - // templates hand out. No object store is wired up behind it. + // testStorageLocation is the --snapshot-location the test server runs + // with. No object store is wired up behind it. testStorageLocation = "gs://fake-fake-fake" testActorJWTIssuer = "https://idp.example.test" @@ -471,9 +471,6 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st Atespace: testAtespace, Name: "tmpl1", }, - SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: testStorageLocation, - }, Containers: containers, Volumes: volumes, WorkerSelector: &ateapi.Selector{ @@ -526,7 +523,7 @@ func newDurableSnapshot(gen int32, owner ateapi.SnapshotOwner, fidelity ateapi.S Fidelity: fidelity, ActorTemplateUid: templateUID, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: uri}, }, } @@ -549,7 +546,7 @@ func newPausedSnapshot(t *testing.T, gen int32, fidelity ateapi.SnapshotFidelity ActorTemplateUid: templateUID, Locality: locality, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING, Object: &ateapi.ObjectSnapshot{SnapshotUri: uri.String()}, }, } @@ -559,7 +556,7 @@ func durableSnapshot(status *ateapi.ActorStatus) *ateapi.Snapshot { var best *ateapi.Snapshot for _, snap := range status.GetSnapshots() { st := snap.GetDurableSnapshot() - if st.GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED && + if st.GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED && st.GetObject().GetSnapshotUri() != "" { if best == nil || snap.GetGeneration() > best.GetGeneration() { best = snap @@ -622,9 +619,6 @@ func createTemplateWithSelector(t *testing.T, tc *testContext, name string, sele Atespace: testAtespace, Name: name, }, - SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: testStorageLocation, - }, Containers: []*ateapi.Container{ {Name: "main", Image: "main@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", Command: []string{"/main"}}, }, diff --git a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go index 2091cf2535..fece94e98e 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go @@ -16,6 +16,7 @@ package functionaltest import ( "context" + "errors" "fmt" "testing" @@ -362,6 +363,30 @@ func TestUpdateTag_NotFound(t *testing.T) { assertGrpcError(t, err, codes.NotFound, "Tag test-atespace/does-not-exist not found") } +// TestUpdateTag_DeletingTag verifies a tag left in place by a failed delete +// can no longer be updated. +func TestUpdateTag_DeletingTag(t *testing.T) { + ns := namespaceForTest("ns-update-tag-deleting") + tc := setupTest(t, ns) + defer tc.cleanup() + ctx := context.Background() + createTemplate(t, tc, ns) + + tagRef := resources.TagRefFromTag(seedTag(t, tc, "actor-1", "v1")) + tc.objectStore.OnList = func(string, string) error { return errors.New("injected list failure") } + if _, err := tc.client.DeleteTag(ctx, &ateapi.DeleteTagRequest{Tag: tagRef.ToObjectRef()}); err == nil { + t.Fatal("DeleteTag succeeded despite the failing snapshot cleanup") + } + + marked, err := tc.client.GetTag(ctx, &ateapi.GetTagRequest{Tag: tagRef.ToObjectRef()}) + if err != nil { + t.Fatalf("GetTag after the failed delete: %v", err) + } + marked.Scope = ateapi.TagScope_TAG_SCOPE_PUBLISHED + _, err = tc.client.UpdateTag(ctx, &ateapi.UpdateTagRequest{Tag: marked}) + assertGrpcError(t, err, codes.FailedPrecondition, "Tag "+tagRef.String()+" is being deleted") +} + func TestDeleteTag(t *testing.T) { ns := namespaceForTest("ns-delete-tag") tc := setupTest(t, ns) diff --git a/cmd/ateapi/internal/controlapi/hosting_test.go b/cmd/ateapi/internal/controlapi/hosting_test.go index e32474d057..3537fae1c1 100644 --- a/cmd/ateapi/internal/controlapi/hosting_test.go +++ b/cmd/ateapi/internal/controlapi/hosting_test.go @@ -235,8 +235,8 @@ func TestHostingNodeAfterResumeOntoNewWorker(t *testing.T) { // TestHostingNodeWithoutWorker pins which no-worker shapes count assigned_node // as hosting the actor. None of them reads the store. func TestHostingNodeWithoutWorker(t *testing.T) { - durable := func(st ateapi.SnapshotStorageStatus) *ateapi.SnapshotStorage { - return &ateapi.SnapshotStorage{Status: st, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://snaps/root/atespaces/a/actors/uid/snap-1"}} + durable := func(st ateapi.SnapshotStorageState) *ateapi.SnapshotStorage { + return &ateapi.SnapshotStorage{State: st, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://snaps/root/atespaces/a/actors/uid/snap-1"}} } status := func(state ateapi.ActorState, locality string, d *ateapi.SnapshotStorage) *ateapi.ActorStatus { return &ateapi.ActorStatus{ @@ -246,9 +246,9 @@ func TestHostingNodeWithoutWorker(t *testing.T) { Snapshots: []*ateapi.Snapshot{{Uuid: "snap-1", Generation: 1, Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, Locality: locality, DurableSnapshot: d}}, } } - olderOnNode := status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + olderOnNode := status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)) olderOnNode.LastAssignedGeneration = 2 - olderOnNode.Snapshots = append(olderOnNode.Snapshots, &ateapi.Snapshot{Uuid: "snap-2", Generation: 2, Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, DurableSnapshot: durable(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)}) + olderOnNode.Snapshots = append(olderOnNode.Snapshots, &ateapi.Snapshot{Uuid: "snap-2", Generation: 2, Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, DurableSnapshot: durable(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)}) noAssignedNode := status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", nil) noAssignedNode.AssignedNode = "" for _, tc := range []struct { @@ -257,11 +257,11 @@ func TestHostingNodeWithoutWorker(t *testing.T) { want string }{ {"paused", status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", nil), "node-1"}, - {"paused, upload pending", status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)), "node-1"}, - {"paused, uploaded, local copy kept", status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)), "node-1"}, + {"paused, upload pending", status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)), "node-1"}, + {"paused, uploaded, local copy kept", status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)), "node-1"}, {"older snapshot on the node, latest not", olderOnNode, "node-1"}, {"local copy but no assigned node", noAssignedNode, ""}, - {"suspending without a worker", status(ateapi.ActorState_ACTOR_STATE_SUSPENDING, "node-1", durable(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)), ""}, + {"suspending without a worker", status(ateapi.ActorState_ACTOR_STATE_SUSPENDING, "node-1", durable(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)), ""}, {"crashed with a local copy", status(ateapi.ActorState_ACTOR_STATE_CRASHED, "node-1", nil), ""}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers.go b/cmd/ateapi/internal/controlapi/snapshot_helpers.go index 3e29b78312..1735536310 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers.go @@ -53,10 +53,10 @@ func latestSnapshot(status *ateapi.ActorStatus, matches func(*ateapi.Snapshot) b } // findLatestDurableSnapshot returns the highest-generation Snapshot on status -// whose DurableSnapshot matches storageStatus, or nil if none exists. -func findLatestDurableSnapshot(status *ateapi.ActorStatus, storageStatus ateapi.SnapshotStorageStatus) *ateapi.Snapshot { +// whose DurableSnapshot matches storageState, or nil if none exists. +func findLatestDurableSnapshot(status *ateapi.ActorStatus, storageState ateapi.SnapshotStorageState) *ateapi.Snapshot { return latestSnapshot(status, func(snap *ateapi.Snapshot) bool { - return snap.GetDurableSnapshot().GetStatus() == storageStatus + return snap.GetDurableSnapshot().GetState() == storageState }) } @@ -107,7 +107,7 @@ func pruneSnapshots(status *ateapi.ActorStatus, local *ateapi.Snapshot) { if status == nil { return } - latestDurable := findLatestDurableSnapshot(status, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + latestDurable := findLatestDurableSnapshot(status, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) kept := status.Snapshots[:0] for _, snap := range status.Snapshots { if snap != local { @@ -121,7 +121,7 @@ func pruneSnapshots(status *ateapi.ActorStatus, local *ateapi.Snapshot) { } // newDurableSnapshot constructs a Snapshot with DurableSnapshot populated. -func newDurableSnapshot(gen int32, owner ateapi.SnapshotOwner, fidelity ateapi.SnapshotFidelity, templateUID, uuid, uri string, storageStatus ateapi.SnapshotStorageStatus) *ateapi.Snapshot { +func newDurableSnapshot(gen int32, owner ateapi.SnapshotOwner, fidelity ateapi.SnapshotFidelity, templateUID, uuid, uri string, storageState ateapi.SnapshotStorageState) *ateapi.Snapshot { return &ateapi.Snapshot{ Uuid: uuid, Generation: gen, @@ -129,18 +129,18 @@ func newDurableSnapshot(gen int32, owner ateapi.SnapshotOwner, fidelity ateapi.S Fidelity: fidelity, ActorTemplateUid: templateUID, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: storageStatus, + State: storageState, Object: &ateapi.ObjectSnapshot{SnapshotUri: uri}, }, } } // tagDurableSnapshotURI returns the durable snapshot URI on tag matching -// storageStatus (or any status if storageStatus is UNSPECIFIED), or "" if none +// storageState (or any state if storageState is UNSPECIFIED), or "" if none // exists. -func tagDurableSnapshotURI(tag *ateapi.Tag, storageStatus ateapi.SnapshotStorageStatus) string { +func tagDurableSnapshotURI(tag *ateapi.Tag, storageState ateapi.SnapshotStorageState) string { st := tag.GetStatus().GetSnapshot().GetDurableSnapshot() - if storageStatus != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED && st.GetStatus() != storageStatus { + if storageState != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED && st.GetState() != storageState { return "" } return st.GetObject().GetSnapshotUri() diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go index 760cb85a3f..791bcbeccb 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go @@ -34,7 +34,7 @@ func TestFindSnapshotByGenerationAndSnapshotAtLatestGeneration(t *testing.T) { } s1 := newPausedSnapshot(t, 1, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-1", "node-1") - s2 := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + s2 := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) status := &ateapi.ActorStatus{ LastAssignedGeneration: 2, Snapshots: []*ateapi.Snapshot{s1, s2}, @@ -52,16 +52,16 @@ func TestFindSnapshotByGenerationAndSnapshotAtLatestGeneration(t *testing.T) { } func TestFindLatestDurableAndLocalSnapshots(t *testing.T) { - if snap := findLatestDurableSnapshot(nil, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); snap != nil { + if snap := findLatestDurableSnapshot(nil, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED); snap != nil { t.Errorf("findLatestDurableSnapshot(nil) = %v, want nil", snap) } if snap := findLatestLocalSnapshot(nil); snap != nil { t.Errorf("findLatestLocalSnapshot(nil) = %v, want nil", snap) } - s1 := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) - s2 := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) - s3 := newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + s1 := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) + s2 := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) + s3 := newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) s4 := newPausedSnapshot(t, 4, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-4", "node-1") // Place s2 before s1 to verify generation comparison rather than slice order. @@ -69,17 +69,17 @@ func TestFindLatestDurableAndLocalSnapshots(t *testing.T) { Snapshots: []*ateapi.Snapshot{s2, s1, s3, s4}, } - gotSnap := findLatestDurableSnapshot(status, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(status, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if gotSnap != s2 || gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != "gs://b/snap-2" { t.Errorf("findLatestDurableSnapshot(COMPLETED) = %v, want gen 2 (gs://b/snap-2)", gotSnap) } // An async suspend snapshot's durable copy is PENDING until atelet uploads it. - gotSnap = findLatestDurableSnapshot(status, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + gotSnap = findLatestDurableSnapshot(status, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) if gotSnap != s4 { t.Errorf("findLatestDurableSnapshot(PENDING) = %v, want gen 4", gotSnap) } - gotSnap = findLatestDurableSnapshot(&ateapi.ActorStatus{Snapshots: []*ateapi.Snapshot{s2, s1, s3}}, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + gotSnap = findLatestDurableSnapshot(&ateapi.ActorStatus{Snapshots: []*ateapi.Snapshot{s2, s1, s3}}, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) if gotSnap != s3 || gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != "gs://b/snap-3" { t.Errorf("findLatestDurableSnapshot(PENDING) without the paused snapshot = %v, want gen 3 (gs://b/snap-3)", gotSnap) } @@ -114,7 +114,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { }) t.Run("pause flow: pruneSnapshots keeps the latest COMPLETED durable and the new local snapshot", func(t *testing.T) { - completedGen1 := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + completedGen1 := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) completedGen1.Locality = "node-1" localGen2 := newPausedSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-2") localGen3 := newPausedSnapshot(t, 3, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "") @@ -128,7 +128,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { pruneSnapshots(status, localGen3) want := []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), newPausedSnapshot(t, 3, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "node-3"), } if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { @@ -140,12 +140,12 @@ func TestSnapshotPruningHelpers(t *testing.T) { }) t.Run("pruneSnapshots drops a PENDING snapshot that still names a node", func(t *testing.T) { - pendingWithLocality := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + pendingWithLocality := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) pendingWithLocality.Locality = "node-1" local := newPausedSnapshot(t, 3, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "node-1") status := &ateapi.ActorStatus{ Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), pendingWithLocality, local, }, @@ -154,7 +154,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { pruneSnapshots(status, local) want := []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), newPausedSnapshot(t, 3, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "node-1"), } if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { @@ -166,20 +166,20 @@ func TestSnapshotPruningHelpers(t *testing.T) { }) t.Run("revert flow: pruneSnapshots drops all PENDING snapshots and retains latest COMPLETED", func(t *testing.T) { - completedWithLocality := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + completedWithLocality := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) completedWithLocality.Locality = "node-1" status := &ateapi.ActorStatus{ Snapshots: []*ateapi.Snapshot{ completedWithLocality, newPausedSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-2"), - newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, } pruneSnapshots(status, nil) want := []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) @@ -189,9 +189,9 @@ func TestSnapshotPruningHelpers(t *testing.T) { t.Run("suspend flow: pruneSnapshots drops older COMPLETED and PENDING snapshots", func(t *testing.T) { status := &ateapi.ActorStatus{ Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), newPausedSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-1"), - newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), }, } status.Snapshots[2].Locality = "node-1" @@ -199,7 +199,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { pruneSnapshots(status, nil) want := []*ateapi.Snapshot{ - newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) @@ -214,12 +214,12 @@ func TestLatestSnapshotIsLocal(t *testing.T) { local := func(gen int32) *ateapi.Snapshot { return newPausedSnapshot(t, gen, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "node-1") } - withCopy := func(st ateapi.SnapshotStorageStatus) *ateapi.Snapshot { + withCopy := func(st ateapi.SnapshotStorageState) *ateapi.Snapshot { snap := local(1) - snap.DurableSnapshot = &ateapi.SnapshotStorage{Status: st, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://b/up"}} + snap.DurableSnapshot = &ateapi.SnapshotStorage{State: st, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://b/up"}} return snap } - durable := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "durable", "gs://b/durable", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + durable := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "durable", "gs://b/durable", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) notCaptured := newPausedSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "") tests := []struct { @@ -230,8 +230,8 @@ func TestLatestSnapshotIsLocal(t *testing.T) { {"nil status", nil, false}, {"no snapshots", &ateapi.ActorStatus{}, false}, {"local only", &ateapi.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{local(1)}}, true}, - {"local and uploaded", &ateapi.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{withCopy(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)}}, true}, - {"local with an upload pending", &ateapi.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{withCopy(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)}}, true}, + {"local and uploaded", &ateapi.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{withCopy(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)}}, true}, + {"local with an upload pending", &ateapi.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{withCopy(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)}}, true}, {"durable only", &ateapi.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapi.Snapshot{durable}}, false}, {"older local under a pause in progress", &ateapi.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapi.Snapshot{local(1), notCaptured}}, false}, } @@ -245,22 +245,22 @@ func TestLatestSnapshotIsLocal(t *testing.T) { } func TestTagDurableSnapshotURI(t *testing.T) { - if got := tagDurableSnapshotURI(nil, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED); got != "" { + if got := tagDurableSnapshotURI(nil, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED); got != "" { t.Errorf("tagDurableSnapshotURI(nil) = %q, want empty", got) } pendingTag := &ateapi.Tag{ Status: &ateapi.TagStatus{ - Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/tag-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/tag-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, } - if got := tagDurableSnapshotURI(pendingTag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != "" { + if got := tagDurableSnapshotURI(pendingTag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED); got != "" { t.Errorf("tagDurableSnapshotURI(pendingTag, COMPLETED) = %q, want empty", got) } - if got := tagDurableSnapshotURI(pendingTag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING); got != "gs://b/tag-1" { + if got := tagDurableSnapshotURI(pendingTag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING); got != "gs://b/tag-1" { t.Errorf("tagDurableSnapshotURI(pendingTag, PENDING) = %q, want gs://b/tag-1", got) } - if got := tagDurableSnapshotURI(pendingTag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED); got != "gs://b/tag-1" { + if got := tagDurableSnapshotURI(pendingTag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED); got != "gs://b/tag-1" { t.Errorf("tagDurableSnapshotURI(pendingTag, UNSPECIFIED) = %q, want gs://b/tag-1", got) } } diff --git a/cmd/ateapi/internal/controlapi/snapshot_report.go b/cmd/ateapi/internal/controlapi/snapshot_report.go index b03b5ece68..e9c9693e9d 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_report.go +++ b/cmd/ateapi/internal/controlapi/snapshot_report.go @@ -165,7 +165,7 @@ func applyLatestDurable(ctx context.Context, status *ateapi.ActorStatus, reporte if reported == nil { return nil } - if latest := findLatestDurableSnapshot(status, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); latest.GetGeneration() >= reported.GetGeneration() { + if latest := findLatestDurableSnapshot(status, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED); latest.GetGeneration() >= reported.GetGeneration() { if latest.GetGeneration() > reported.GetGeneration() { slog.InfoContext(ctx, msgDurableOlder, slog.Int("generation", int(reported.GetGeneration())), @@ -208,7 +208,7 @@ func applyLatestDurable(ctx context.Context, status *ateapi.ActorStatus, reporte return nil } snap.Fidelity = reported.GetFidelity() - snap.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + snap.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED return nil } @@ -222,7 +222,7 @@ func applyLatestDurable(ctx context.Context, status *ateapi.ActorStatus, reporte func applyDetach(status *ateapi.ActorStatus, node string, upTo int32) error { deleted := func(s *ateapi.Snapshot) bool { return s.GetLocality() == node && s.GetGeneration() <= upTo } if latest := latestSnapshot(status, deleted); status.GetState() != ateapi.ActorState_ACTOR_STATE_CRASHED && latest != nil && - latest.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + latest.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { return apierror.FailedPrecondition("snapshot generation %d on node %s is not durable", latest.GetGeneration(), node) } for _, snap := range status.GetSnapshots() { diff --git a/cmd/ateapi/internal/controlapi/snapshot_report_test.go b/cmd/ateapi/internal/controlapi/snapshot_report_test.go index 2536dde820..ab041533a2 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_report_test.go +++ b/cmd/ateapi/internal/controlapi/snapshot_report_test.go @@ -51,12 +51,12 @@ func ownedURI(t *testing.T, actorUID string, gen int32) string { } // withDurable returns a Snapshot held on locality ("" for none) with a durable -// copy in the given storage status. -func withDurable(t *testing.T, gen int32, locality string, storage ateapi.SnapshotStorageStatus) *ateapi.Snapshot { +// copy in the given storage state. +func withDurable(t *testing.T, gen int32, locality string, storage ateapi.SnapshotStorageState) *ateapi.Snapshot { t.Helper() snap := &ateapi.Snapshot{Uuid: snapName(gen), Generation: gen, Locality: locality, Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY} snap.DurableSnapshot = &ateapi.SnapshotStorage{ - Status: storage, + State: storage, Object: &ateapi.ObjectSnapshot{SnapshotUri: ownedURI(t, reportActorUID, gen)}, } return snap @@ -79,8 +79,8 @@ func reportedAt(t *testing.T, gen int32, uri string) *ateapi.SnapshotReport { func TestApplyActorSnapshotState(t *testing.T) { const ( - pending = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - completed = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pending = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING + completed = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED suspended = ateapi.ActorState_ACTOR_STATE_SUSPENDED running = ateapi.ActorState_ACTOR_STATE_RUNNING ) @@ -325,8 +325,8 @@ func TestApplyActorSnapshotState(t *testing.T) { // and one that is applied, aren't logged. func TestApplyLatestDurableLogsStaleReports(t *testing.T) { const ( - pending = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - completed = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pending = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING + completed = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED ) tests := []struct { name string @@ -433,7 +433,7 @@ func TestApplyReportedSnapshotState(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.AssignedNode = reportNode s.LastAssignedGeneration = 2 - snap := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", pending.Name(), pending.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + snap := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", pending.Name(), pending.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) snap.Locality = reportNode s.Snapshots = []*ateapi.Snapshot{snap} }) @@ -509,7 +509,7 @@ func TestApplyReportedSnapshotState(t *testing.T) { t.Run("applied to a "+st.String()+" actor", func(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.State = st - s.Snapshots[0].DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + s.Snapshots[0].DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING }) if err := w.ApplyReportedSnapshotState(ctx, actorRef, uid, reportNode, state); err != nil { t.Fatalf("ApplyReportedSnapshotState() failed: %v", err) @@ -518,7 +518,7 @@ func TestApplyReportedSnapshotState(t *testing.T) { if err != nil { t.Fatal(err) } - if st := got.GetStatus().GetSnapshots()[0].GetDurableSnapshot().GetStatus(); st != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + if st := got.GetStatus().GetSnapshots()[0].GetDurableSnapshot().GetState(); st != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { t.Errorf("durable snapshot status = %v, want COMPLETED", st) } @@ -533,7 +533,7 @@ func TestApplyReportedSnapshotState(t *testing.T) { // A suspend that crashed before it finalized leaves its snapshot on no node. older := mustActorSnapshotURI(t, tmpl, actor, "snap-1") - snapshot := func(gen int32, uri resources.SnapshotURI, locality string, storage ateapi.SnapshotStorageStatus) *ateapi.Snapshot { + snapshot := func(gen int32, uri resources.SnapshotURI, locality string, storage ateapi.SnapshotStorageState) *ateapi.Snapshot { snap := newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uri.Name(), uri.String(), storage) snap.Locality = locality return snap @@ -546,35 +546,35 @@ func TestApplyReportedSnapshotState(t *testing.T) { }{{ name: "a crashed suspend's upload stays pending", snapshots: []*ateapi.Snapshot{ - snapshot(1, older, reportNode, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - snapshot(2, pending, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + snapshot(1, older, reportNode, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + snapshot(2, pending, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, state: state, want: []*ateapi.Snapshot{ - snapshot(1, older, reportNode, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - snapshot(2, pending, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + snapshot(1, older, reportNode, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + snapshot(2, pending, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, { name: "a crashed suspend's detach releases the older snapshot", snapshots: []*ateapi.Snapshot{ - snapshot(1, older, reportNode, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - snapshot(2, pending, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + snapshot(1, older, reportNode, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + snapshot(2, pending, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, state: &ateapi.ActorSnapshotState{LatestDurable: uploaded}, want: []*ateapi.Snapshot{ - snapshot(1, older, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - snapshot(2, pending, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + snapshot(1, older, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + snapshot(2, pending, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, { name: "a crashed suspend's detach releases an older snapshot that isn't durable", snapshots: []*ateapi.Snapshot{ - snapshot(1, older, reportNode, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), - snapshot(2, pending, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + snapshot(1, older, reportNode, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), + snapshot(2, pending, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, state: &ateapi.ActorSnapshotState{LatestDurable: uploaded}, want: []*ateapi.Snapshot{ - snapshot(1, older, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), - snapshot(2, pending, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + snapshot(1, older, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), + snapshot(2, pending, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }} { t.Run(tc.name, func(t *testing.T) { @@ -601,7 +601,7 @@ func TestActorOnNode(t *testing.T) { actor := func(assigned string, localities ...string) *ateapi.Actor { status := &ateapi.ActorStatus{AssignedNode: assigned} for i, l := range localities { - status.Snapshots = append(status.Snapshots, withDurable(t, int32(i+1), l, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)) + status.Snapshots = append(status.Snapshots, withDurable(t, int32(i+1), l, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)) } return &ateapi.Actor{Status: status} } @@ -635,8 +635,8 @@ func ateletReported(t *testing.T, gen int32) *ateletpb.SnapshotReport { func TestApplyAteletResponseState(t *testing.T) { const ( - pending = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - completed = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pending = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING + completed = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED suspended = ateapi.ActorState_ACTOR_STATE_SUSPENDED suspending = ateapi.ActorState_ACTOR_STATE_SUSPENDING diff --git a/cmd/ateapi/internal/controlapi/tag.go b/cmd/ateapi/internal/controlapi/tag.go index ec9362c707..6fdc1fa015 100644 --- a/cmd/ateapi/internal/controlapi/tag.go +++ b/cmd/ateapi/internal/controlapi/tag.go @@ -100,10 +100,14 @@ func (s *ServiceImpl) ListTags(ctx context.Context, atespace string, opts store. return s.store.ListTags(ctx, atespace, opts) } -// errTagPending is what the update's mutate closure returns when the stored -// tag's create never finished, so the caller can tell it apart from a store -// failure and answer FAILED_PRECONDITION. -var errTagPending = errors.New("tag is still being created") +// errTagPending and errTagDeleting are what the update's mutate closure +// returns when the stored tag's create never finished or its delete has +// started, so the caller can tell them apart from a store failure and answer +// FAILED_PRECONDITION. +var ( + errTagPending = errors.New("tag is still being created") + errTagDeleting = errors.New("tag is being deleted") +) func (s *RPCService) UpdateTag(ctx context.Context, req *ateapi.UpdateTagRequest) (*ateapi.Tag, error) { // First scrub any fields that users are not allowed to set. @@ -120,10 +124,13 @@ func (s *RPCService) UpdateTag(ctx context.Context, req *ateapi.UpdateTagRequest tagRef := resources.TagRefFromTag(in) storedTag, err := s.impl.UpdateTag(ctx, tagRef, store.PreconditionFrom(in), func(toUpdate *ateapi.Tag) error { + if toUpdate.GetMetadata().GetDeleteTime() != nil { + return errTagDeleting + } // A tag whose create never finished names a partial copy. Publishing it // — or changing its scope at all — would hand out content that is still // being written, or may never be. - if tagDurableSnapshotURI(toUpdate, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { + if tagDurableSnapshotURI(toUpdate, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) == "" { return errTagPending } // Metadata and status are server-owned fields. @@ -146,6 +153,9 @@ func (s *RPCService) UpdateTag(ctx context.Context, req *ateapi.UpdateTagRequest if errors.Is(err, errTagPending) { return nil, apierror.FailedPrecondition("Tag %s/%s is still being created", tagRef.Atespace, tagRef.Name) } + if errors.Is(err, errTagDeleting) { + return nil, apierror.FailedPrecondition("Tag %s/%s is being deleted", tagRef.Atespace, tagRef.Name) + } if errors.Is(err, store.ErrImmutableField) { return nil, apierror.InvalidArgument("while updating tag %s/%s: %v", tagRef.Atespace, tagRef.Name, err) } diff --git a/cmd/ateapi/internal/controlapi/tag_test.go b/cmd/ateapi/internal/controlapi/tag_test.go index a79c3cc000..9bd244092a 100644 --- a/cmd/ateapi/internal/controlapi/tag_test.go +++ b/cmd/ateapi/internal/controlapi/tag_test.go @@ -135,7 +135,7 @@ func TestUpdateTag(t *testing.T) { req: &ateapi.Tag{ Scope: ateapi.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapi.TagStatus{ - Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, "", "elsewhere", "gs://attacker/elsewhere", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, "", "elsewhere", "gs://attacker/elsewhere", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), ActorTemplateUid: "other-template-uid", }, }, @@ -251,7 +251,7 @@ func newTestSuspendedActor(t *testing.T, ctx context.Context, st store.Interface return mustUpdateActorStatus(t, ctx, st, actor, func(status *ateapi.ActorStatus) { status.LastAssignedGeneration = 1 status.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uri.Name(), uri.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uri.Name(), uri.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } }) } @@ -266,13 +266,13 @@ func newTestTag(t *testing.T, name string, actor *ateapi.Actor) *ateapi.Tag { if err != nil { t.Fatalf("NewTagSnapshotURI: %v", err) } - actorSnap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + actorSnap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) return &ateapi.Tag{ Metadata: &ateapi.ResourceMetadata{Atespace: atespace, Name: name}, Scope: ateapi.TagScope_TAG_SCOPE_ATESPACE, SourceActor: resources.ActorRefFromActor(actor).ToObjectRef(), Status: &ateapi.TagStatus{ - Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, actorSnap.GetFidelity(), actorSnap.GetActorTemplateUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, actorSnap.GetFidelity(), actorSnap.GetActorTemplateUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), }, } } @@ -283,7 +283,7 @@ func newTestTag(t *testing.T, name string, actor *ateapi.Actor) *ateapi.Tag { func newPendingTestTag(t *testing.T, name string, actor *ateapi.Actor) *ateapi.Tag { t.Helper() tag := newTestTag(t, name, actor) - tag.Status.Snapshot.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + tag.Status.Snapshot.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING return tag } diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 285649afd2..a2e5395956 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -197,8 +197,11 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource } return 0, err } - // TODO: Skip a template with delete_time set. DeleteActorTemplate may - // already have deleted the golden actor and tag, and this pass rebuilds them. + // DeleteActorTemplate may already have deleted the golden actor and tag, + // and this pass would rebuild them. + if tmpl.GetMetadata().GetDeleteTime() != nil { + return 0, nil + } if tmpl.GetSnapshotConfig().GetGoldenSnapshotConfig().GetMode() == ateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED { return 0, nil @@ -236,7 +239,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource if tag.GetStatus().GetActorTemplateUid() != tmpl.GetMetadata().GetUid() || resources.ActorRefFromObjectRef(tag.GetSourceActor()) != resources.ActorRefFromObjectRef(goldenActorRef) { return 0, r.fail(ctx, tmpl, reasonGoldenTagConflict, "golden tag belongs to another actor or template") } - if tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) != "" { + if tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) != "" { return 0, r.saveGoldenTag(ctx, tmpl, goldenActorRef) } // CreateTag cannot resume an incomplete copy. Delete it before retrying. @@ -256,7 +259,11 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource switch state := actor.GetStatus().GetState(); state { case ateapi.ActorState_ACTOR_STATE_CRASHED: - return 0, r.fail(ctx, tmpl, reasonGoldenActorCrashed, "golden actor crashed before its snapshot was taken") + msg := "golden actor crashed before its snapshot was taken" + if crash := actor.GetStatus().GetCrash().GetMessage(); crash != "" { + msg += ": " + crash + } + return 0, r.fail(ctx, tmpl, reasonGoldenActorCrashed, msg) case ateapi.ActorState_ACTOR_STATE_RUNNING: takeAt := goldenSnapshotStatus.GetSnapshotTime() @@ -296,7 +303,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource ateapi.ActorState_ACTOR_STATE_SUSPENDED: // The golden actor was never resumed, or a previous resume didn't // finish; ResumeActor is reentrant from both. - if findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) != nil { + if findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) != nil { // Golden actors never start from a source snapshot, so an // existing snapshot means an earlier suspend completed // without being recorded. @@ -340,7 +347,7 @@ func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *a // A crash during suspend is observed as CRASHED on the retry. return fmt.Errorf("while suspending golden actor: %w", err) } - if findLatestDurableSnapshot(resp.GetActor().GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == nil { + if findLatestDurableSnapshot(resp.GetActor().GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) == nil { return fmt.Errorf("suspending golden actor produced no external snapshot") } return nil diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index 68c3ca2778..5abd08e491 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -166,6 +166,7 @@ type fakeGoldenControl struct { exists bool goldenState ateapi.ActorState goldenSnapshot string + goldenCrash string // snapshot is the external snapshot a completed suspend produces; empty // simulates a suspend that wrote none. snapshot string @@ -185,10 +186,13 @@ func (c *fakeGoldenControl) CreateAtespace(_ context.Context, req *ateapi.Create func (c *fakeGoldenControl) goldenActorStatus() *ateapi.ActorStatus { st := &ateapi.ActorStatus{State: c.goldenState} + if c.goldenCrash != "" { + st.Crash = &ateapi.ActorCrash{Message: c.goldenCrash} + } if c.goldenSnapshot != "" { st.LastAssignedGeneration = 1 st.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "golden-snapshot", c.goldenSnapshot, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "golden-snapshot", c.goldenSnapshot, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } } return st @@ -283,7 +287,7 @@ func (c *fakeGoldenControl) CreateTag(_ context.Context, req *ateapi.CreateTagRe c.tag = proto.CloneOf(req.GetTag()) c.tag.Status = &ateapi.TagStatus{ ActorTemplateUid: testTemplateUID, - Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "golden-snapshot", c.goldenSnapshot, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "golden-snapshot", c.goldenSnapshot, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } return proto.CloneOf(c.tag), nil } @@ -352,6 +356,10 @@ func withoutWakeupProbe(tmpl *ateapi.ActorTemplate) { } } +func withDeleteTime(tmpl *ateapi.ActorTemplate) { + tmpl.Metadata.DeleteTime = timestamppb.Now() +} + func withoutGoldenSnapshot(tmpl *ateapi.ActorTemplate) { tmpl.SnapshotConfig.GoldenSnapshotConfig.Mode = ateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED } @@ -530,9 +538,9 @@ func TestReconcileOne(t *testing.T) { { name: "crashed golden actor fails the template", template: testTemplate(), - control: &fakeGoldenControl{exists: true, goldenState: ateapi.ActorState_ACTOR_STATE_CRASHED}, + control: &fakeGoldenControl{exists: true, goldenState: ateapi.ActorState_ACTOR_STATE_CRASHED, goldenCrash: "atelet Run: unhandled tar entry"}, wantFailedReason: reasonGoldenActorCrashed, - wantMessage: "crashed", + wantMessage: "crashed before its snapshot was taken: atelet Run: unhandled tar entry", }, { name: "resume failure requeues without failing", @@ -578,6 +586,12 @@ func TestReconcileOne(t *testing.T) { name: "deleted template is a noop", control: &fakeGoldenControl{}, }, + { + name: "template marked for deletion is a noop", + template: testTemplate(withDeleteTime), + control: &fakeGoldenControl{}, + wantUnchanged: true, + }, { name: "terminal golden snapshot is a noop", template: testTemplate(withGoldenTag()), @@ -1129,7 +1143,7 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { Scope: ateapi.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapi.TagStatus{ ActorTemplateUid: testTemplateUID, - Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "tag-snapshot", "gs://bucket/tag-snapshot", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "tag-snapshot", "gs://bucket/tag-snapshot", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), }, } incomplete := proto.CloneOf(completed) @@ -1187,7 +1201,7 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { if !proto.Equal(st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus().GetGoldenTag(), ref) { t.Fatal("golden tag not recorded") } - if tagDurableSnapshotURI(control.tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { + if tagDurableSnapshotURI(control.tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) == "" { t.Fatal("golden tag has no snapshot") } if len(control.createReqs) != 0 || len(control.resumeReqs) != 0 || len(control.suspendReqs) != 0 { diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index 3cf41f8899..deea2f51ee 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -407,7 +407,7 @@ func (w *ActorWorkflow) ensureExternalSnapshotsReleased(ctx context.Context, act // wrote anything. func actorSnapshotStoragePrefix(actor *ateapi.Actor) (resources.StoragePrefix, error) { actorOwner := actorSnapshotOwner(actor) - if snap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); snap != nil { + if snap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED); snap != nil { snapshotURI := snap.GetDurableSnapshot().GetObject().GetSnapshotUri() uri, err := resources.ParseSnapshotURI(snapshotURI) if err != nil { @@ -423,7 +423,7 @@ func actorSnapshotStoragePrefix(actor *ateapi.Actor) (resources.StoragePrefix, e // Nothing of the actor's own is recorded. Unless a suspend died partway, // nothing was ever written under its prefix: the in-progress URI is // recorded before atelet uploads the first object. - inProgressSnap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + inProgressSnap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) inProgress := inProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() if inProgress == "" { return resources.StoragePrefix{}, nil diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index 54151f9ebd..ed75f10c5a 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -154,7 +154,7 @@ func (*detachDeletePlugin) DeleteVolume(context.Context, string) error { return // count as detached, and any other error fails the step as before. func TestRevertAndDelete_AssignedNodeGone(t *testing.T) { memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED for _, tc := range []struct { name string state ateapi.ActorState @@ -262,7 +262,7 @@ func TestRevertAndDelete_AssignedNodeGone(t *testing.T) { if s := got.GetStatus(); s.GetState() != ateapi.ActorState_ACTOR_STATE_SUSPENDED || s.GetAssignedNode() != "" { t.Errorf("after revert: state %v, assigned node %q, want SUSPENDED with no node", s.GetState(), s.GetAssignedNode()) } - if snaps := got.GetStatus().GetSnapshots(); len(snaps) != 1 || snaps[0].GetGeneration() != 1 || snaps[0].GetDurableSnapshot().GetStatus() != completed { + if snaps := got.GetStatus().GetSnapshots(); len(snaps) != 1 || snaps[0].GetGeneration() != 1 || snaps[0].GetDurableSnapshot().GetState() != completed { t.Errorf("after revert: snapshots %v, want only the completed durable snapshot at generation 1", snaps) } default: @@ -385,8 +385,8 @@ func TestEnsureExternalSnapshotsReleased(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 2 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, currentType, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, currentType, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) @@ -434,7 +434,7 @@ func TestEnsureExternalSnapshotsReleased_CollectsStrandedSnapshots(t *testing.T) actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", currentSnapshot.Name(), currentSnapshot.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", currentSnapshot.Name(), currentSnapshot.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } }) @@ -474,7 +474,7 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) { mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) @@ -507,7 +507,7 @@ func TestEnsureExternalSnapshotsReleased_DeletePrefixFailure(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } }) @@ -585,7 +585,7 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } }) } @@ -657,7 +657,7 @@ func TestDeleteActorWorkflow_RetryKeepsAdmission(t *testing.T) { observed := mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) // The guards a client sends after reading the actor. diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 8f43df3dfe..5ae65943ce 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -407,12 +407,6 @@ func (w *ActorWorkflow) validateAssignedWorker(ctx context.Context, actorRef res return worker, nil } -// admittedResources is what an assignment books against its worker, or nil -// when the actor declared no limits and so reserves nothing. -func admittedResources(constraints scheduling.Constraints) *ateapi.Resources { - return constraints.Limits -} - // workerHoldingStaleClaim recovers a claim written before the Actor update. // It releases the claim if the Worker is no longer eligible. func (w *ActorWorkflow) workerHoldingStaleClaim(ctx context.Context, actor *ateapi.Actor, constraints scheduling.Constraints, relaxable bool) (*ateapi.Worker, error) { @@ -474,7 +468,7 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour }() constraints := schedulingConstraints(actor, actorTemplate) - snapRuntime := constraints.RequiredRuntime + snapRuntime := constraints.SnapshotRuntime class = constraints.SandboxClass relaxable := canRelaxRuntime(actor, constraints) @@ -514,9 +508,15 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour Name: actor.GetMetadata().GetName(), }, ActorUid: actor.GetMetadata().GetUid(), - // Record what this claim reserves so release returns the same amount. - Resources: admittedResources(constraints), } + // Record what this claim books so release returns the same amount. It + // includes the overhead the cached Worker reports, which admit checks + // again under the row lock. + booking, ok := scheduling.Booking(assignedWorker, constraints) + if !ok { + return nil, nil, errWorkerFilledUp + } + assignment.Resources = booking // The candidate came from a watch-fed cache, so it may already be full or no // longer eligible. The store re-asks under the Worker's row lock, where the @@ -528,6 +528,11 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour if !w.scheduler.Applies(fresh, constraints) || !w.scheduler.HasRoom(fresh, constraints) { return errWorkerFilledUp } + // The Worker re-registered with a different overhead since the cache + // saw it. Rare, and the retry reads the new one. + if got, _ := scheduling.Booking(fresh, constraints); !proto.Equal(got, booking) { + return errWorkerFilledUp + } bound = fresh return nil } @@ -537,8 +542,9 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour return nil, nil, fmt.Errorf("selected worker disappeared before claim: %w", store.ErrVersionConflict) } // errWorkerFilledUp leaves the cache alone: it reports no room for - // this actor's size, so forgetting the worker would hide it from - // every smaller actor until the next watch event or relist. + // this actor's size, or an overhead the cache hasn't seen yet, so + // forgetting the worker would hide it from every smaller actor until + // the next watch event or relist. return nil, nil, err } @@ -623,9 +629,11 @@ func schedulingConstraints(actor *ateapi.Actor, tmpl *ateapi.ActorTemplate) sche if node := actor.GetStatus().GetAssignedNode(); node != "" { requiredNodes = []string{node} } + required := hardware.NewRequirement(requiredSnapshotRuntime(snap, tmpl)) c := scheduling.Constraints{ SandboxClass: snap.GetSandboxRuntime().GetSandboxClass(), - RequiredRuntime: hardware.NewRequirement(requiredSnapshotRuntime(snap, tmpl)), + RequiredRuntime: required, + SnapshotRuntime: required, ActorSelector: labels.SelectorFromSet(labels.Set(actor.GetWorkerSelector().GetMatchLabels())), RequiredNodes: requiredNodes, Limits: tmpl.GetResources().GetLimits(), @@ -641,7 +649,7 @@ func schedulingConstraints(actor *ateapi.Actor, tmpl *ateapi.ActorTemplate) sche // creation included), or nil when the actor has none and cold-boots. func snapshotToResume(status *ateapi.ActorStatus) *ateapi.Snapshot { return latestSnapshot(status, func(s *ateapi.Snapshot) bool { - return s.GetLocality() != "" || s.GetDurableSnapshot().GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return s.GetLocality() != "" || s.GetDurableSnapshot().GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED }) } @@ -683,7 +691,7 @@ func canRelaxRuntime(actor *ateapi.Actor, constraints scheduling.Constraints) bo if localElsewhere(snap, status.GetAssignedNode()) { return true } - return snap.GetDurableSnapshot().GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return snap.GetDurableSnapshot().GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } // appliesWithFallback reports whether worker satisfies constraints, or — when diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_detach.go b/cmd/ateapi/internal/controlapi/workflow_resume_detach.go index 3b9495995c..174d9a374d 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_detach.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_detach.go @@ -198,7 +198,7 @@ func localElsewhere(snap *ateapi.Snapshot, node string) bool { // no completed durable copy. func latestLocalNotDurable(actor *ateapi.Actor) bool { local := findLatestLocalSnapshot(actor.GetStatus()) - return local != nil && local.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return local != nil && local.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } func canDetachWithoutAtelet(ctx context.Context, actor *ateapi.Actor, err error) bool { @@ -217,7 +217,7 @@ func canDetachWithoutAtelet(ctx context.Context, actor *ateapi.Actor, err error) return false } local := findLatestLocalSnapshot(actor.GetStatus()) - return local != nil && local.GetDurableSnapshot().GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return local != nil && local.GetDurableSnapshot().GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } // detachActorError maps a DetachActor error to the one the resume fails with. diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go index e4848bcd57..9479df65e6 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go @@ -362,7 +362,7 @@ func TestDetachFromFullNode(t *testing.T) { snap := newPausedSnapshot(t, 1, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uuid, tt.node) snap.SandboxRuntime = tt.snapRuntime if tt.completed { - snap.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + snap.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } actor := seedActorWithSnapshot(t, ctx, st, suspended, tt.node, snap) w, atelet := newDetachTestWorkflow(t, ctx, st) @@ -390,7 +390,7 @@ func TestDetachFromFullNode(t *testing.T) { if err != nil { t.Errorf("GetActor: %v", err) } else if _, err := st.UpdateActor(ctx, actorRef, store.PreconditionFrom(cur), func(a *ateapi.Actor) error { - a.Status.Snapshots[0].DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + a.Status.Snapshots[0].DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED return nil }); err != nil { t.Errorf("UpdateActor: %v", err) @@ -481,7 +481,7 @@ func TestDetachFromFullNode(t *testing.T) { } got := after.GetStatus().GetSnapshots() if len(got) != 1 || got[0].GetLocality() != "" || - got[0].GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + got[0].GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { t.Errorf("snapshots = %v, want %s durable only and COMPLETED", got, uuid) } }) @@ -611,7 +611,7 @@ func TestResumeActor_NodeGoneWithPendingUpload(t *testing.T) { worker := seedRuntimeWorker(t, ctx, st, "pod-2", "node-2", workerRuntime) memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY durable := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", "durable-1", - someActorSnapshotURI(t, testStorageLocation, "team-a", "durable-1"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + someActorSnapshotURI(t, testStorageLocation, "team-a", "durable-1"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) pending := newPausedSnapshot(t, 2, memory, "", "pending-2", "node-3") if tt.otherRuntime { pending.SandboxRuntime = testRuntime("amd64", "") @@ -647,7 +647,7 @@ func TestResumeActor_NodeGoneWithPendingUpload(t *testing.T) { } // The restore proved the upload complete. snap := findSnapshotByGeneration(s.GetSnapshots(), 2) - if snap.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED || snap.GetLocality() != "" { + if snap.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED || snap.GetLocality() != "" { t.Errorf("generation 2 = %v, want COMPLETED with no locality", snap) } return @@ -749,7 +749,7 @@ func TestDetachFromFullNode_ReentryRecordsAfterConflict(t *testing.T) { t.Errorf("AssignedNode = %q, want none", got) } if got := stored.GetStatus().GetSnapshots(); len(got) != 1 || got[0].GetLocality() != "" || - got[0].GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + got[0].GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { t.Errorf("snapshots = %v, want pause-snap-1 durable only and COMPLETED", got) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 6409e679c3..5da1080bbc 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -255,18 +255,18 @@ func TestFinalizeRunning_UnrecordableUploadStillRuns(t *testing.T) { if got.GetStatus().GetState() != ateapi.ActorState_ACTOR_STATE_RUNNING { t.Errorf("state = %v, want RUNNING", got.GetStatus().GetState()) } - if snap := got.GetStatus().GetSnapshots()[0]; snap.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING || snap.GetLocality() != "node-3" { + if snap := got.GetStatus().GetSnapshots()[0]; snap.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING || snap.GetLocality() != "node-3" { t.Errorf("snapshot = %v, want it unrecorded: PENDING on node-3", snap) } } func TestRecordRestoredUpload(t *testing.T) { memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - snapshot := func(locality string, fidelity ateapi.SnapshotFidelity, storage ateapi.SnapshotStorageStatus) *ateapi.Snapshot { + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + snapshot := func(locality string, fidelity ateapi.SnapshotFidelity, storage ateapi.SnapshotStorageState) *ateapi.Snapshot { snap := newPausedSnapshot(t, 1, fidelity, "", "snap-1", locality) - snap.DurableSnapshot.Status = storage + snap.DurableSnapshot.State = storage return snap } for _, tt := range []struct { @@ -1436,7 +1436,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { unspecScope := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED extSnap := func(uri string, fidelity ateapi.SnapshotFidelity) *ateapi.Snapshot { - return newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", durableSnapshotName, uri, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + return newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", durableSnapshotName, uri, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) } localSnap := func(name string, fidelity ateapi.SnapshotFidelity) *ateapi.Snapshot { return newPausedSnapshot(t, 1, fidelity, "", name, "node-1") @@ -1451,7 +1451,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { uploadedLocalSnap := func(name string, fidelity ateapi.SnapshotFidelity) *ateapi.Snapshot { snap := localSnap(name, fidelity) snap.DurableSnapshot = &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: pauseUploadURI}, } return snap @@ -1525,7 +1525,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, } stampedExtSnap := func(owner ateapi.SnapshotOwner, uri string, fidelity ateapi.SnapshotFidelity, rt *ateapi.ActorSandboxRuntime) *ateapi.Snapshot { - snap := newDurableSnapshot(1, owner, fidelity, "", durableSnapshotName, uri, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap := newDurableSnapshot(1, owner, fidelity, "", durableSnapshotName, uri, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) snap.SandboxRuntime = rt return snap } @@ -2707,7 +2707,7 @@ func seedRuntimeWorker(t *testing.T, ctx context.Context, st store.Interface, po // generation 1 under team-a/id1's own prefix, stamped with runtime. func durableMemorySnapshot(t *testing.T, runtime *ateapi.ActorSandboxRuntime) *ateapi.Snapshot { t.Helper() - snap := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-1"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-1"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) snap.SandboxRuntime = runtime return snap } @@ -3005,7 +3005,7 @@ func TestValidateAssignedWorker_RuntimeFallback(t *testing.T) { // Re-entry must accept a previously allowed ROOTFS assignment. local := newPausedSnapshot(t, 1, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-arm") local.SandboxRuntime = testSandboxRuntime() - local.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + local.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED resumingActor := &ateapi.Actor{ Metadata: &ateapi.ResourceMetadata{Atespace: "team-a", Name: "id1", Uid: "own-actor-uid"}, Status: &ateapi.ActorStatus{ @@ -3047,11 +3047,11 @@ func TestRequiredSnapshotRuntime(t *testing.T) { memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY rootfs := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS volumes := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING - durable := func(gen int32, fidelity ateapi.SnapshotFidelity, capturedUnder string, status ateapi.SnapshotStorageStatus, rt *ateapi.ActorSandboxRuntime) *ateapi.Snapshot { - snap := newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, capturedUnder, "snap", "gs://bucket/snap", status) + durable := func(gen int32, fidelity ateapi.SnapshotFidelity, capturedUnder string, state ateapi.SnapshotStorageState, rt *ateapi.ActorSandboxRuntime) *ateapi.Snapshot { + snap := newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, capturedUnder, "snap", "gs://bucket/snap", state) snap.SandboxRuntime = rt return snap } @@ -3333,8 +3333,8 @@ func TestResume_ChecksMountedVolumesBeforeAttach(t *testing.T) { func TestCanRelaxRuntime(t *testing.T) { rt := testSandboxRuntime() memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING suspended := ateapi.ActorState_ACTOR_STATE_SUSPENDED resuming := ateapi.ActorState_ACTOR_STATE_RESUMING @@ -3344,9 +3344,9 @@ func TestCanRelaxRuntime(t *testing.T) { local := func(gen int32) *ateapi.Snapshot { return newPausedSnapshot(t, gen, memory, "", "local", "node-1") } - localDurable := func(gen int32, storageStatus ateapi.SnapshotStorageStatus) *ateapi.Snapshot { + localDurable := func(gen int32, storageState ateapi.SnapshotStorageState) *ateapi.Snapshot { snap := local(gen) - snap.GetDurableSnapshot().Status = storageStatus + snap.GetDurableSnapshot().State = storageState return snap } localNoDurable := func(gen int32) *ateapi.Snapshot { @@ -3578,3 +3578,27 @@ func TestAssignWorkerAttempt_WorkerWithoutNodeRejectsVolumes(t *testing.T) { }) } } + +// A Worker that re-registered with a different overhead since the cache saw +// it is refused under its row lock, so what the assignment records is what +// admit checked. +func TestAssignWorkerAttempt_RefusesChangedOverhead(t *testing.T) { + cached := contendedWorker("pod-a") + cached.Status.SandboxClasses[0].DefaultRuntime.ActorOverhead = &ateapi.RuntimeOverhead{MemoryBytes: 4 << 20} + sched := &filledUpScheduler{workers: []*ateapi.Worker{cached}} + w, actor, _ := newContendedWorkflow(t, sched) + locked := proto.Clone(cached).(*ateapi.Worker) + locked.Status.SandboxClasses[0].DefaultRuntime.ActorOverhead = &ateapi.RuntimeOverhead{MemoryBytes: 16 << 20} + w.store.(*contentionStore).workers[locked.GetMetadata().GetName()] = locked + + tmpl := &ateapi.ActorTemplate{Resources: &ateapi.ResourceRequirements{Limits: &ateapi.Resources{MemoryBytes: 256 << 20}}} + _, _, err := w.assignWorkerAttempt(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, tmpl) + if !errors.Is(err, errWorkerFilledUp) { + t.Fatalf("assignWorkerAttempt() = %v, want %v", err, errWorkerFilledUp) + } + + w.store.(*contentionStore).workers[locked.GetMetadata().GetName()] = cached + if _, _, err := w.assignWorkerAttempt(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, tmpl); err != nil { + t.Fatalf("assignWorkerAttempt() with an unchanged overhead = %v", err) + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_revert.go b/cmd/ateapi/internal/controlapi/workflow_revert.go index 6c553fcd91..5c5ff604be 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert.go @@ -220,7 +220,7 @@ func (w *ActorWorkflow) ensureInProgressSnapshotDiscarded(ctx context.Context, a var inProgress []string for _, snap := range actor.GetStatus().GetSnapshots() { storage := snap.GetDurableSnapshot() - if storage.GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING && storage.GetObject().GetSnapshotUri() != "" { + if storage.GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING && storage.GetObject().GetSnapshotUri() != "" { inProgress = append(inProgress, storage.GetObject().GetSnapshotUri()) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_revert_test.go b/cmd/ateapi/internal/controlapi/workflow_revert_test.go index 40d247d296..e03e8eb48f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert_test.go @@ -146,7 +146,7 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { inProgress.DurableSnapshot.Object.SnapshotUri = mustActorSnapshotURI(t, tmpl, actor, "local-in-progress").String() s.LastAssignedGeneration = 3 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "kept", keptURI, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "kept", keptURI, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), local1, inProgress, } @@ -170,7 +170,7 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { if got := gotStatus.GetLastAssignedGeneration(); got != 3 { t.Errorf("last_assigned_generation = %d, want 3", got) } - gotSnap := findLatestDurableSnapshot(gotStatus, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(gotStatus, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if got := gotSnap.GetGeneration(); got != 1 { t.Errorf("retained snapshot generation = %d, want 1", got) } @@ -183,7 +183,7 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { if got := findLatestLocalSnapshot(gotStatus); got != nil { t.Errorf("local snapshot info = %v, want nil", got) } - if got := findLatestDurableSnapshot(gotStatus, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING); got != nil { + if got := findLatestDurableSnapshot(gotStatus, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING); got != nil { t.Errorf("in-progress durable snapshot = %v, want nil", got) } if got := len(gotStatus.GetSnapshots()); got != 1 { @@ -260,12 +260,12 @@ func TestRevertActor_UnreachableNode(t *testing.T) { s.AssignedNode = tt.node s.LastAssignedGeneration = 2 snap := newPausedSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "uploaded", tt.node) - snap.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + snap.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED if tt.pending { - snap.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + snap.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING } s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, snap.GetFidelity(), "", "older", "gs://snapshots/atespaces/team-a/actors/older/snapshots/snapshot", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, snap.GetFidelity(), "", "older", "gs://snapshots/atespaces/team-a/actors/older/snapshots/snapshot", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), snap, } if tt.assigned { @@ -343,7 +343,7 @@ func TestRevertActor_UnreachableNode(t *testing.T) { // terminate, as in TestRevertActor_ReturnsActorToItsSnapshot. func TestRevertActor_PausedSnapshotAlreadyUploaded(t *testing.T) { memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED ctx := context.Background() st, cleanup := storetest.SetupTestStore(t) defer cleanup() @@ -365,7 +365,7 @@ func TestRevertActor_PausedSnapshotAlreadyUploaded(t *testing.T) { objects.PutSnapshot(t, uploaded, "manifest.json") mustUpdateActorStatus(t, ctx, st, actor, func(s *ateapi.ActorStatus) { paused := newPausedSnapshot(t, 2, memory, "", "pause-snap", "node-1") - paused.DurableSnapshot = &ateapi.SnapshotStorage{Status: completed, Object: &ateapi.ObjectSnapshot{SnapshotUri: uploaded.String()}} + paused.DurableSnapshot = &ateapi.SnapshotStorage{State: completed, Object: &ateapi.ObjectSnapshot{SnapshotUri: uploaded.String()}} s.LastAssignedGeneration = 2 s.Snapshots = []*ateapi.Snapshot{ newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", old.Name(), old.String(), completed), @@ -472,7 +472,7 @@ func TestRevertActor_NoSnapshotToRevertTo(t *testing.T) { if got := reverted.GetStatus().GetCrash(); got != nil { t.Errorf("Crash = %v, want cleared by the revert", got) } - gotSnap := findLatestDurableSnapshot(reverted.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(reverted.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != "" { t.Errorf("external snapshot = %q, want none", got) } @@ -581,15 +581,15 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, currentType, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, currentType, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } if tt.earlierPause { s.LastAssignedGeneration++ - s.Snapshots = append(s.Snapshots, newDurableSnapshot(s.LastAssignedGeneration, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", earlierPause.Name(), earlierPause.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)) + s.Snapshots = append(s.Snapshots, newDurableSnapshot(s.LastAssignedGeneration, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", earlierPause.Name(), earlierPause.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)) } if tt.inFlight != "" { s.LastAssignedGeneration++ - s.Snapshots = append(s.Snapshots, newDurableSnapshot(s.LastAssignedGeneration, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)) + s.Snapshots = append(s.Snapshots, newDurableSnapshot(s.LastAssignedGeneration, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)) } }) @@ -647,7 +647,7 @@ func TestEnsureRevertedFinalized_NoObjectStore(t *testing.T) { AssignedNode: "node-1", LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "abandoned", someActorSnapshotURI(t, testStorageLocation, "team-a", "abandoned"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "abandoned", someActorSnapshotURI(t, testStorageLocation, "team-a", "abandoned"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, }) @@ -667,7 +667,7 @@ func TestEnsureRevertedFinalized_NoObjectStore(t *testing.T) { if got := finalized.GetStatus().GetAssignedNode(); got != "" { t.Errorf("assigned node = %q, want empty", got) } - if got := findLatestDurableSnapshot(finalized.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING); got != nil { + if got := findLatestDurableSnapshot(finalized.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING); got != nil { t.Errorf("in-progress durable snapshot = %v, want nil", got) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index b0a2b801ef..6c9d814fcd 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -194,7 +194,7 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res actorTemplate.GetMetadata().GetUid(), uri.Name(), uri.String(), - ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING, ) snap.SandboxRuntime = proto.CloneOf(toUpdate.GetStatus().GetWorkerAssignment().GetSandboxRuntime()) toUpdate.Status.Snapshots = append(toUpdate.Status.Snapshots, snap) @@ -404,9 +404,9 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r toUpdate.Status.Snapshots = applied.GetSnapshots() snap := snapshotAtLatestGeneration(toUpdate.Status) if mode == suspendSync { - snap.GetDurableSnapshot().Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + snap.GetDurableSnapshot().State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } - durable := snap.GetDurableSnapshot().GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + durable := snap.GetDurableSnapshot().GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED // Keep the actor on the node that holds its snapshot. atelet holds an // async suspend's snapshot until it detaches it, which it does only // once the snapshot is durable. @@ -453,14 +453,14 @@ func ateletKeptLocal(state *ateletpb.ActorSnapshotState, snap *ateapi.Snapshot) // whole prefix. An upload atelet reports can make a newer snapshot COMPLETED // outside any workflow, so the next one can drop more than one. func (w *ActorWorkflow) releaseDroppedSnapshots(ctx context.Context, before, after *ateapi.Actor) { - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED kept := map[string]bool{} for _, snap := range after.GetStatus().GetSnapshots() { kept[snap.GetDurableSnapshot().GetObject().GetSnapshotUri()] = true } for _, snap := range before.GetStatus().GetSnapshots() { dropped := snap.GetDurableSnapshot().GetObject().GetSnapshotUri() - if snap.GetDurableSnapshot().GetStatus() != completed || kept[dropped] { + if snap.GetDurableSnapshot().GetState() != completed || kept[dropped] { continue } if err := w.releaseReplacedSnapshot(ctx, before, dropped); err != nil { diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go index ca62090ba6..9031284cfe 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go @@ -240,7 +240,7 @@ func TestSuspendActorAsync_Finalized_RecordsFidelity(t *testing.T) { if localSnap.GetFidelity() != tc.want { t.Errorf("local Snapshot.Fidelity = %v, want %v", localSnap.GetFidelity(), tc.want) } - if got := localSnap.GetDurableSnapshot().GetStatus(); got != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { + if got := localSnap.GetDurableSnapshot().GetState(); got != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING { t.Errorf("local DurableSnapshot.Status = %v, want PENDING", got) } if got := localSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got == "" { @@ -373,7 +373,7 @@ func TestSuspendActorAsync_AteletPaused_DialFailureLeavesActorRetryable(t *testi var snapshots []*ateapi.Snapshot if tt.prevSnapshot != "" { - snapshots = append(snapshots, newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "prev-snapshot", tt.prevSnapshot, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + snapshots = append(snapshots, newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "prev-snapshot", tt.prevSnapshot, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)) } snapshots = append(snapshots, newInProgressSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "actor-1-never-written")) @@ -405,7 +405,7 @@ func TestSuspendActorAsync_AteletPaused_DialFailureLeavesActorRetryable(t *testi if gotSnap := snapshotAtLatestGeneration(stored.GetStatus()); gotSnap.GetUuid() != "actor-1-never-written" { t.Errorf("in-progress local snapshot uuid = %q, want preserved for debugging", gotSnap.GetUuid()) } - gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != tt.prevSnapshot { t.Errorf("SnapshotUri = %q, want %q", got, tt.prevSnapshot) } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 7eb8474e4a..82acf9f239 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -109,7 +109,7 @@ func TestSuspendActor_CheckpointError(t *testing.T) { t.Errorf("worker allocated actors = %d, want 1", got) } pending := snapshotAtLatestGeneration(st) - if pending.GetUuid() == "" || pending.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { + if pending.GetUuid() == "" || pending.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING { t.Fatalf("snapshot = %v, want a persisted pending snapshot", pending) } first := atelet.checkpointRequest().GetSnapshot() @@ -170,7 +170,7 @@ func TestSuspendActor_MarkedSuspending_SnapshotURI(t *testing.T) { assertStoppedAtDial(t, err) marked := mustReadActor(t, ctx, persistence, actorRef) - inProgressSnap := findLatestDurableSnapshot(marked.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + inProgressSnap := findLatestDurableSnapshot(marked.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) uri := mustParseSnapshotURI(t, inProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri()) if !resources.IsValidResourceName(uri.Name()) { t.Errorf("in-progress snapshot name = %q, want a valid resource name", uri.Name()) @@ -194,15 +194,15 @@ func TestSuspendActor_MarkedSuspending_SnapshotURI(t *testing.T) { // the same shape, so the retry's mode may differ from the first attempt's. func TestSuspendActor_MarkedSuspending_ReentryKeepsPersistedSnapshotLocation(t *testing.T) { const firstAttempt = "gs://bucket/root/atespaces/team-a/actors/actor-uid/snapshots/first-attempt" - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED tests := []struct { name string mode suspendMode wantUploadMode ateletpb.UploadMode wantLocality string - wantDurable ateapi.SnapshotStorageStatus + wantDurable ateapi.SnapshotStorageState wantNode string }{ {"sync", suspendSync, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE, "", completed, ""}, @@ -246,7 +246,7 @@ func TestSuspendActor_MarkedSuspending_ReentryKeepsPersistedSnapshotLocation(t * if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != firstAttempt { t.Errorf("committed durable snapshot URI = %q, want the first attempt's location", got) } - if got := gotSnap.GetDurableSnapshot().GetStatus(); got != tt.wantDurable { + if got := gotSnap.GetDurableSnapshot().GetState(); got != tt.wantDurable { t.Errorf("durable copy status = %v, want %v", got, tt.wantDurable) } if got := gotSnap.GetLocality(); got != tt.wantLocality { @@ -268,7 +268,7 @@ func TestSuspendActorWorkflow_SuspendedReturnsAsIs(t *testing.T) { withDurable := func(a *ateapi.Actor) { a.Status.LastAssignedGeneration = 1 a.Status.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "durable-snap", durableURI, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "durable-snap", durableURI, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } } @@ -338,7 +338,7 @@ func TestSuspendActor_MarkedSuspending_StateMatrix(t *testing.T) { } if seedState == ateapi.ActorState_ACTOR_STATE_RUNNING { latest := snapshotAtLatestGeneration(stored.GetStatus()) - if latest.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING || latest.GetLocality() != "" { + if latest.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING || latest.GetLocality() != "" { t.Errorf("state %v: latest snapshot = %v, want a PENDING durable copy and no locality", seedState, latest) } } @@ -431,9 +431,9 @@ func TestSuspendActor_AteletSuspended_DialFailureLeavesActorRetryable(t *testing seedSuspendTemplate(t, ctx, persistence, &ateapi.SnapshotConfig{StorageLocation: testStorageLocation}) var snapshots []*ateapi.Snapshot if tt.prevSnapshot != "" { - snapshots = append(snapshots, newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "prev-snapshot", tt.prevSnapshot, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + snapshots = append(snapshots, newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "prev-snapshot", tt.prevSnapshot, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)) } - snapshots = append(snapshots, newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "never-written", neverWritten, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)) + snapshots = append(snapshots, newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "never-written", neverWritten, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)) actor := &ateapi.Actor{ Metadata: &ateapi.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, @@ -466,10 +466,10 @@ func TestSuspendActor_AteletSuspended_DialFailureLeavesActorRetryable(t *testing if stored.GetStatus().GetState() != ateapi.ActorState_ACTOR_STATE_SUSPENDING { t.Errorf("state = %v, want unchanged SUSPENDING", stored.GetStatus().GetState()) } - if gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING); gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != neverWritten { + if gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING); gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != neverWritten { t.Errorf("in-progress durable snapshot URI = %q, want preserved for debugging", gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri()) } - gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != tt.prevSnapshot { t.Errorf("SnapshotUri = %q, want %q", got, tt.prevSnapshot) } @@ -500,7 +500,7 @@ func TestSuspendActor_AteletSuspended_WireRequest(t *testing.T) { WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-uid-1", snapshotName, snapshotURI, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-uid-1", snapshotName, snapshotURI, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, }) @@ -588,8 +588,8 @@ func TestSuspendActor_Finalized_ReleasesReplacedSnapshot(t *testing.T) { mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 2 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, prevType, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, prevType, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) @@ -599,7 +599,7 @@ func TestSuspendActor_Finalized_ReleasesReplacedSnapshot(t *testing.T) { } // Whichever way the previous snapshot went, the actor now owns the // one it just wrote and is no longer borrowing. - gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != fresh.String() { t.Errorf("external snapshot = %q, want the one this suspend wrote, %q", got, fresh) } @@ -621,8 +621,8 @@ func TestSuspendActor_Finalized_ReleasesReplacedSnapshot(t *testing.T) { func TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot(t *testing.T) { const node = "node-1" memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING for _, tc := range []struct { name string // reported records the upload before the suspend, as atelet's report @@ -732,8 +732,8 @@ func TestSnapshotLocationChange_LeavesNothingBehind(t *testing.T) { mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 2 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) @@ -797,8 +797,8 @@ func TestSuspendActor_Finalized_CommitsDespiteObjectStoreFailure(t *testing.T) { mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 2 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) @@ -810,11 +810,11 @@ func TestSuspendActor_Finalized_CommitsDespiteObjectStoreFailure(t *testing.T) { if got := stored.GetStatus().GetState(); got != ateapi.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got) } - gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != fresh.String() { t.Errorf("actor snapshot uri = %q, want the one this suspend wrote, %q", got, fresh) } - if got := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING); got != nil { + if got := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING); got != nil { t.Errorf("in-progress durable snapshot = %v, want cleared", got) } if len(objects.Snapshot(t, previous)) == 0 { @@ -855,8 +855,8 @@ func TestSuspendActor_Finalized_KeepsReplacedSnapshotOnConflict(t *testing.T) { mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 2 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) @@ -902,7 +902,7 @@ func TestSuspendActor_Finalized_KeepsCrashFromWorkerDelete(t *testing.T) { WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, }) @@ -957,7 +957,7 @@ func TestSuspendActor_Finalized_ReleasesOnlyOwnWorker(t *testing.T) { WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, } @@ -1022,7 +1022,7 @@ func TestSuspendActor_Finalized_MissingDurableSnapshotClearsAssignment(t *testin WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, }) @@ -1113,7 +1113,7 @@ type snapshotOutcome struct { fidelity ateapi.SnapshotFidelity locality string // durable is the durable copy's status, UNSPECIFIED when there is none. - durable ateapi.SnapshotStorageStatus + durable ateapi.SnapshotStorageState } // TestSuspendAndPause_CharacteristicMatrix pins, for each actor shape either @@ -1123,15 +1123,15 @@ type snapshotOutcome struct { func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { const node = "node-1" volumes := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES - durable := func(gen int32, status ateapi.SnapshotStorageStatus) *ateapi.Snapshot { + durable := func(gen int32, state ateapi.SnapshotStorageState) *ateapi.Snapshot { name := fmt.Sprintf("snap-%d", gen) - return newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, volumes, "", name, someActorSnapshotURI(t, testStorageLocation, "team-a", name), status) + return newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, volumes, "", name, someActorSnapshotURI(t, testStorageLocation, "team-a", name), state) } local := func(gen int32, locality string) *ateapi.Snapshot { return newPausedSnapshot(t, gen, volumes, "", fmt.Sprintf("snap-%d", gen), locality) } - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING ateletVolumes := ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES immediate := ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE @@ -1385,7 +1385,7 @@ func describeSnapshots(status *ateapi.ActorStatus) []snapshotOutcome { generation: snap.GetGeneration(), fidelity: snap.GetFidelity(), locality: snap.GetLocality(), - durable: snap.GetDurableSnapshot().GetStatus(), + durable: snap.GetDurableSnapshot().GetState(), }) } return out @@ -1423,11 +1423,11 @@ func assertRequestNamesLatestSnapshot(t *testing.T, atelet *capturingAtelet, tmp func TestSuspendActor_AppliesCheckpointState(t *testing.T) { const node = "node-1" memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - durable := func(gen int32, status ateapi.SnapshotStorageStatus) *ateapi.Snapshot { + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + durable := func(gen int32, state ateapi.SnapshotStorageState) *ateapi.Snapshot { name := fmt.Sprintf("snap-%d", gen) - return newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", name, someActorSnapshotURI(t, testStorageLocation, "team-a", name), status) + return newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", name, someActorSnapshotURI(t, testStorageLocation, "team-a", name), state) } report := func(gen int32, uri string, fidelity ateletpb.SnapshotFidelity) *ateletpb.SnapshotReport { return &ateletpb.SnapshotReport{Snapshot: &ateletpb.Snapshot{ @@ -1616,7 +1616,7 @@ func TestSuspendActor_AppliesCheckpointState(t *testing.T) { func TestSuspendActor_Finalized_RecordsLocalCopyWhenKept(t *testing.T) { const node = "node-1" memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED localReport := func(snap *ateletpb.Snapshot) *ateletpb.ActorSnapshotState { return &ateletpb.ActorSnapshotState{LatestLocal: &ateletpb.SnapshotReport{Snapshot: snap}} } diff --git a/cmd/ateapi/internal/controlapi/workflow_tag.go b/cmd/ateapi/internal/controlapi/workflow_tag.go index 25e50e2c8e..ba20448362 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag.go @@ -194,7 +194,7 @@ func (w *ActorWorkflow) ensureTagSnapshotReleased(ctx context.Context, tag *atea return nil } tagRef := resources.TagRefFromTag(tag) - snapshotURI := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED) + snapshotURI := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED) if snapshotURI == "" { markSkipped(ctx, "the tag records no snapshot") return nil @@ -293,7 +293,7 @@ func (w *ActorWorkflow) ensureTagSourceSnapshotDurable(ctx context.Context, acto defer func() { err = done(err) }() snap := latestSnapshot(actor.GetStatus(), func(*ateapi.Snapshot) bool { return true }) - if snap.GetDurableSnapshot().GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + if snap.GetDurableSnapshot().GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { markSkipped(ctx, "the actor's latest snapshot is already durable") return actor, nil } @@ -387,7 +387,7 @@ func promoteSnapshotError(ctx context.Context, err error) error { // sourceSnapshotForTag returns the actor's latest COMPLETED external snapshot, // the one a tag copies. func sourceSnapshotForTag(actorRef resources.ActorRef, actor *ateapi.Actor) (*ateapi.Snapshot, error) { - snap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if snap == nil { return nil, apierror.FailedPrecondition("Actor %s holds no external snapshot to tag", actorRef) } @@ -465,7 +465,7 @@ func (w *ActorWorkflow) ensureTagSnapshotRecorded(ctx context.Context, tag *atea "", dst.Name(), dst.String(), - ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING, ) pending.SandboxRuntime = proto.CloneOf(snapshot.GetSandboxRuntime()) stored, err := w.store.UpdateTag(ctx, tagRef, store.PreconditionFrom(tag), func(toUpdate *ateapi.Tag) error { @@ -523,7 +523,7 @@ func (w *ActorWorkflow) ensureTagFinalized(ctx context.Context, tag *ateapi.Tag, "", dst.Name(), dst.String(), - ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, ) finalSnapshot.SandboxRuntime = proto.CloneOf(snapshot.GetSandboxRuntime()) stored, err := w.store.UpdateTag(ctx, tagRef, store.PreconditionFrom(tag), func(toUpdate *ateapi.Tag) error { diff --git a/cmd/ateapi/internal/controlapi/workflow_tag_test.go b/cmd/ateapi/internal/controlapi/workflow_tag_test.go index b1e14ec14c..e82a75213e 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag_test.go @@ -50,7 +50,7 @@ func seedTagSource(t *testing.T, ctx context.Context, persistence store.Interfac objects.PutSnapshot(t, uri, objectNames...) actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 - snap := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, template.GetMetadata().GetUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, template.GetMetadata().GetUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) snap.SandboxRuntime = testSandboxRuntime() s.Snapshots = []*ateapi.Snapshot{snap} }) @@ -94,7 +94,7 @@ func TestTagActorSnapshot(t *testing.T) { t.Fatalf("TagActorSnapshot: %v", err) } - tagSnapshot := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + tagSnapshot := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if tagSnapshot == "" || tagSnapshot == actorSnapshot.String() { t.Fatalf("tag snapshot uri = %q, want a copy of its own rather than the actor's %q", tagSnapshot, actorSnapshot) } @@ -214,7 +214,7 @@ func TestTagActorSnapshot_Preconditions(t *testing.T) { } s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, uri.Name(), uri.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, uri.Name(), uri.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } }) } @@ -274,7 +274,7 @@ func TestTagActorSnapshot_TagsNewestExternalSnapshot(t *testing.T) { uri := mustActorSnapshotURI(t, template, actor, "actor-1-snapshot") objects.PutSnapshot(t, uri, "manifest.json") mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { - snap := newDurableSnapshot(tt.extGen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, template.GetMetadata().GetUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap := newDurableSnapshot(tt.extGen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, template.GetMetadata().GetUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) snap.Locality = tt.locality s.LastAssignedGeneration = tt.lastAssigned s.Snapshots = []*ateapi.Snapshot{snap} @@ -458,9 +458,9 @@ func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { }) pausedURI := mustActorSnapshotURI(t, template, actor, "paused") objects.PutSnapshot(t, pausedURI, "manifest.json", "memory.zst") - pausedStatus := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + pausedStatus := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING if tt.uploaded { - pausedStatus = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pausedStatus = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } paused := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, pausedURI.Name(), pausedURI.String(), pausedStatus) if !tt.nodeLost { @@ -474,7 +474,7 @@ func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { if tt.olderDurable { olderURI := mustActorSnapshotURI(t, template, actor, "older") objects.PutSnapshot(t, olderURI, "manifest.json") - s.Snapshots = append(s.Snapshots, newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, olderURI.Name(), olderURI.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + s.Snapshots = append(s.Snapshots, newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, olderURI.Name(), olderURI.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)) } s.Snapshots = append(s.Snapshots, proto.CloneOf(paused)) }) @@ -554,7 +554,7 @@ func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { return } - tagSnapshot := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + tagSnapshot := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if tt.nodeLost { // Nothing to upload from: the actor is left as it was, and the // tag copied the older durable snapshot. @@ -569,7 +569,7 @@ func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { // The local snapshot is now durable and stays on its node. wantPaused := proto.CloneOf(paused) - wantPaused.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + wantPaused.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED if diff := cmp.Diff(wantPaused, findSnapshotByGeneration(stored.GetStatus().GetSnapshots(), 2), protocmp.Transform()); diff != "" { t.Errorf("the actor's latest snapshot mismatch (-want +got):\n%s", diff) } @@ -612,7 +612,7 @@ func TestTagActorSnapshot_RecreateAfterCopyFailure(t *testing.T) { if err != nil { t.Fatalf("GetTag after the external store failure: %v", err) } - if got := tagDurableSnapshotURI(pending, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != "" { + if got := tagDurableSnapshotURI(pending, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED); got != "" { t.Errorf("snapshot uri after the failure = %q, want unset: the copy never finished", got) } strandedURI := mustReservedTagSnapshotURI(t, pending) @@ -651,7 +651,7 @@ func TestTagActorSnapshot_RecreateAfterCopyFailure(t *testing.T) { if err != nil { t.Fatalf("TagActorSnapshot after the delete: %v", err) } - recreated := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + recreated := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if recreated == "" || recreated == stranded { t.Fatalf("snapshot uri after the delete = %q, want a fresh prefix rather than the stranded %q", recreated, stranded) } @@ -732,7 +732,7 @@ func TestTagActorSnapshot_RecreateAfterRecordFailure(t *testing.T) { if err != nil { t.Fatalf("TagActorSnapshot after the delete: %v", err) } - if got, want := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), mustTagSnapshotURI(t, template, "team-a", tag.GetMetadata().GetUid()).String(); got != want { + if got, want := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), mustTagSnapshotURI(t, template, "team-a", tag.GetMetadata().GetUid()).String(); got != want { t.Errorf("recreated tag snapshot uri = %q, want UID-based URI %q", got, want) } } @@ -782,7 +782,7 @@ func TestTagActorSnapshot_RacesDelete(t *testing.T) { // Nothing was collected out from under the copy, and the row in the store it // is still there. - if got := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != pendingURI { + if got := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED); got != pendingURI { t.Errorf("snapshot uri = %q, want the prefix the create reserved, %q", got, pendingURI) } if diff := cmp.Diff([]string{"manifest.json", "memory.zst"}, objects.Snapshot(t, mustParseSnapshotURI(t, pendingURI))); diff != "" { @@ -967,7 +967,7 @@ func TestDeleteTag_NotFound(t *testing.T) { func mustReservedTagSnapshotURI(t *testing.T, tag *ateapi.Tag) resources.SnapshotURI { t.Helper() - uri, err := resources.ParseSnapshotURI(tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED)) + uri, err := resources.ParseSnapshotURI(tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED)) if err != nil { t.Fatalf("ParseSnapshotURI: %v", err) } diff --git a/cmd/ateapi/internal/controlapi/workflow_test.go b/cmd/ateapi/internal/controlapi/workflow_test.go index d8e50ce33b..0c43928fd8 100644 --- a/cmd/ateapi/internal/controlapi/workflow_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_test.go @@ -299,7 +299,7 @@ func TestActorStateChangeRecordSkippedOnConflict(t *testing.T) { if _, err := persistence.UpdateActor(ctx, actorRef, store.PreconditionFrom(stale), func(toUpdate *ateapi.Actor) error { toUpdate.Status.LastAssignedGeneration = 1 toUpdate.Status.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "someone-else", someActorSnapshotURI(t, testStorageLocation, "team-a", "someone-else"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "someone-else", someActorSnapshotURI(t, testStorageLocation, "team-a", "someone-else"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } return nil }); err != nil { diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index 3ac600bba7..ad40c383a6 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -181,7 +181,7 @@ func withUploadedPausedShape(t *testing.T, node string) func(*ateapi.Actor) { paused := withPausedShape(t, node) return func(a *ateapi.Actor) { paused(a) - a.Status.Snapshots[0].DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + a.Status.Snapshots[0].DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } } @@ -190,7 +190,7 @@ func withUploadedPausedShape(t *testing.T, node string) func(*ateapi.Actor) { // yet. func newInProgressSnapshot(t *testing.T, gen int32, fidelity ateapi.SnapshotFidelity, uuid string) *ateapi.Snapshot { t.Helper() - return newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", uuid, someActorSnapshotURI(t, testStorageLocation, "team-a", uuid), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + return newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", uuid, someActorSnapshotURI(t, testStorageLocation, "team-a", uuid), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) } // newPausedSnapshot builds the snapshot an asynchronous suspend leaves on node diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go index aeed776291..afbf559580 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go @@ -115,7 +115,7 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { actor := seedAPIActor(t, ctx, persistence, ateapi.ActorState_ACTOR_STATE_RUNNING, func(a *ateapi.Actor) { a.Status.LastAssignedGeneration = 2 a.Status.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "last", someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "last", someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), newInProgressSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "partial-snapshot"), } }) @@ -140,12 +140,12 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { } // The in-progress checkpoint is kept: it names the prefix whatever atelet // already uploaded lives under, which delete or revert needs to collect it. - gotInProgressSnap := findLatestDurableSnapshot(got.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + gotInProgressSnap := findLatestDurableSnapshot(got.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot"); gotInProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != want { t.Errorf("in-progress external checkpoint not preserved: %v", got.GetStatus()) } // The last completed snapshot is what makes the actor resumable, so it stays. - gotCompletedSnap := findLatestDurableSnapshot(got.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotCompletedSnap := findLatestDurableSnapshot(got.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"); gotCompletedSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != want { t.Errorf("external snapshot = %q, want it preserved as %q", gotCompletedSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(), want) } diff --git a/cmd/ateapi/internal/scheduling/scheduling.go b/cmd/ateapi/internal/scheduling/scheduling.go index d43dd5aef5..0e6acd13c6 100644 --- a/cmd/ateapi/internal/scheduling/scheduling.go +++ b/cmd/ateapi/internal/scheduling/scheduling.go @@ -19,6 +19,8 @@ import ( "context" "errors" "fmt" + "math" + "math/bits" "math/rand" "slices" @@ -44,6 +46,12 @@ type Constraints struct { // asks it of every candidate. RequiredRuntime hardware.Requirement + // SnapshotRuntime is the runtime of the snapshot the actor restores from, + // or the zero Requirement when it starts on a worker's default runtime. + // Unlike RequiredRuntime it is never relaxed: it picks the worker runtime + // the actor runs under, and so the overhead it is booked with. + SnapshotRuntime hardware.Requirement + // TemplateSelector and ActorSelector must both match the worker's labels. TemplateSelector labels.Selector ActorSelector labels.Selector @@ -123,7 +131,11 @@ func (s *scheduler) Schedule(ctx context.Context, constraints Constraints) (*ate if !s.Applies(worker, constraints) { continue } - if cand, ok := checkRoom(worker, constraints.Limits); ok { + booking, ok := Booking(worker, constraints) + if !ok { + continue + } + if cand, ok := checkRoom(worker, booking); ok { candidates = append(candidates, cand) } } @@ -252,10 +264,60 @@ func (s *scheduler) Applies(worker *ateapi.Worker, constraints Constraints) bool // this size. A dimension the worker does not report is unconstrained, so // placement is never blocked by missing data. func (s *scheduler) HasRoom(worker *ateapi.Worker, constraints Constraints) bool { - _, ok := checkRoom(worker, constraints.Limits) + booking, ok := Booking(worker, constraints) + if !ok { + return false + } + _, ok = checkRoom(worker, booking) return ok } +// Booking is what an actor takes from the worker's capacity: its limits plus +// the overhead of the worker runtime it would run under there. It reports +// false when that doesn't fit in an int64, and the actor doesn't fit on the worker. +func Booking(worker *ateapi.Worker, constraints Constraints) (*ateapi.Resources, bool) { + overhead := constraints.SnapshotRuntime.WorkerRuntimeFor(worker.GetStatus()).GetActorOverhead() + if overhead == nil { + return constraints.Limits, true + } + limit := constraints.Limits.GetMemoryBytes() + scaled, ok := perGiB(limit, overhead.GetMemoryBytesPerGib()) + if !ok { + return nil, false + } + memory, ok := sum(limit, overhead.GetMemoryBytes(), scaled) + if !ok { + return nil, false + } + return &ateapi.Resources{CpuMillicores: constraints.Limits.GetCpuMillicores(), MemoryBytes: memory}, true +} + +// perGiB scales rate bytes per GiB to n bytes, and reports false on overflow. +func perGiB(n, rate int64) (int64, bool) { + if n < 0 || rate < 0 { + return 0, false + } + hi, lo := bits.Mul64(uint64(n), uint64(rate)) + // divide the 128-bit product by 1<<30 + if hi>>30 != 0 { + return 0, false + } + v := hi<<34 | lo>>30 + return int64(v), v <= math.MaxInt64 +} + +// sum adds non-negative values, and reports false on overflow. +func sum(vs ...int64) (int64, bool) { + var total int64 + for _, v := range vs { + if v < 0 || v > math.MaxInt64-total { + return 0, false + } + total += v + } + return total, true +} + func checkRoom(worker *ateapi.Worker, want *ateapi.Resources) (candidate, bool) { capacity := worker.GetStatus().GetCapacity() used := worker.GetStatus().GetAllocated() diff --git a/cmd/ateapi/internal/scheduling/scheduling_test.go b/cmd/ateapi/internal/scheduling/scheduling_test.go index 8e2985d52c..b06a1b0c21 100644 --- a/cmd/ateapi/internal/scheduling/scheduling_test.go +++ b/cmd/ateapi/internal/scheduling/scheduling_test.go @@ -17,11 +17,13 @@ package scheduling import ( "context" "errors" + "math" "testing" "go.ate.dev/substrate/api/ateapi/v1" "go.ate.dev/substrate/internal/hardware" "go.ate.dev/substrate/internal/resources" + "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/labels" ) @@ -712,3 +714,87 @@ func TestSchedulePowerOfTwoChoicesShiftedSecondSample(t *testing.T) { t.Fatalf("Schedule() = %q, want %q", got.GetWorkerPod(), "w-2") } } + +// withOverhead sets the actor overhead on the worker's runtimes: def on the +// default, restorable on the restorable ones in order. +func withOverhead(def *ateapi.RuntimeOverhead, restorable ...*ateapi.RuntimeOverhead) func(*ateapi.Worker) { + return func(w *ateapi.Worker) { + c := w.Status.SandboxClasses[0] + c.DefaultRuntime.ActorOverhead = def + for i, o := range restorable { + c.RestorableRuntimes[i].ActorOverhead = o + } + } +} + +func TestBooking(t *testing.T) { + const gib = 1 << 30 + limits := &ateapi.Resources{CpuMillicores: 500, MemoryBytes: gib / 2} + current, old := sandboxRuntime("microvm", "amd64", "new"), sandboxRuntime("microvm", "amd64", "old") + w := worker("w", "", "node-a", nil, + withRuntimes(current, old), + withOverhead(&ateapi.RuntimeOverhead{MemoryBytes: 4 << 20, MemoryBytesPerGib: 8 << 20}, &ateapi.RuntimeOverhead{MemoryBytes: 16 << 20})) + for _, tc := range []struct { + name string + constraints Constraints + want int64 + }{ + {name: "cold boot books the default runtime's", constraints: Constraints{Limits: limits}, want: gib/2 + 4<<20 + 4<<20}, + { + name: "a restore books the matching runtime's", + constraints: Constraints{Limits: limits, RequiredRuntime: hardware.NewRequirement(old), SnapshotRuntime: hardware.NewRequirement(old)}, + want: gib/2 + 16<<20, + }, + { + // relaxing what the worker must match doesn't change the runtime the + // actor restores under, so it doesn't change the price + name: "a relaxed restore still books the matching runtime's", + constraints: Constraints{Limits: limits, SnapshotRuntime: hardware.NewRequirement(old)}, + want: gib/2 + 16<<20, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, ok := Booking(w, tc.constraints) + if !ok || got.GetMemoryBytes() != tc.want || got.GetCpuMillicores() != 500 { + t.Errorf("Booking() = %v, %v, want %d memory bytes and 500 millicores", got, ok, tc.want) + } + }) + } + plain := worker("plain", "gvisor", "node-a", nil) + if got, ok := Booking(plain, Constraints{Limits: limits}); !ok || !proto.Equal(got, limits) { + t.Errorf("Booking() without overhead = %v, %v, want the limits %v", got, ok, limits) + } +} + +// A booking that overflows fits nowhere, rather than wrapping negative and +// fitting everywhere. +func TestBookingOverflow(t *testing.T) { + w := worker("w", "microvm", "node-a", nil, withMaxActors(10), withCapacity(0, 1<<30), + withOverhead(&ateapi.RuntimeOverhead{MemoryBytes: 4 << 20, MemoryBytesPerGib: 8 << 20})) + for _, memory := range []int64{math.MaxInt64, math.MaxInt64 - 1<<20} { + constraints := Constraints{Limits: &ateapi.Resources{MemoryBytes: memory}} + if got, ok := Booking(w, constraints); ok { + t.Errorf("Booking(%d) = %v, want overflow", memory, got) + } + if New(fleet{w}).HasRoom(w, constraints) { + t.Errorf("HasRoom(%d) = true, want false", memory) + } + } +} + +// An actor that fits the worker's free memory alone can still not fit with its +// runtime's overhead. +func TestHasRoomBooksRuntimeOverhead(t *testing.T) { + const gib = 1 << 30 + constraints := Constraints{Limits: &ateapi.Resources{MemoryBytes: gib}} + overhead := withOverhead(&ateapi.RuntimeOverhead{MemoryBytes: 4 << 20}) + tight := worker("tight", "microvm", "node-a", nil, withMaxActors(10), withCapacity(0, gib), overhead) + roomy := worker("roomy", "microvm", "node-a", nil, withMaxActors(10), withCapacity(0, gib+4<<20), overhead) + s := New(fleet{tight, roomy}) + if s.HasRoom(tight, constraints) { + t.Error("HasRoom() = true without room for the actor's overhead, want false") + } + if !s.HasRoom(roomy, constraints) { + t.Error("HasRoom() = false with room for the actor and its overhead, want true") + } +} diff --git a/cmd/ateapi/internal/store/atepg/common_test.go b/cmd/ateapi/internal/store/atepg/common_test.go index 30a21388ba..c25a3ecbfa 100644 --- a/cmd/ateapi/internal/store/atepg/common_test.go +++ b/cmd/ateapi/internal/store/atepg/common_test.go @@ -57,7 +57,7 @@ func setTestPolicyManager(t *testing.T, p *Persistence) { t.Cleanup(fgaServer.Close) _, policyManager, err := authz.New(t.Context(), p.pool, fgaServer, nil, authz.SystemPrincipals{ ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEIDs: []string{installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace)}, IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), diff --git a/cmd/ateapi/internal/store/atepg/tag.go b/cmd/ateapi/internal/store/atepg/tag.go index d5cca85516..2028fb93a9 100644 --- a/cmd/ateapi/internal/store/atepg/tag.go +++ b/cmd/ateapi/internal/store/atepg/tag.go @@ -195,12 +195,12 @@ func isAllowedTagSnapshotTransition(stored, mutated *ateapi.Snapshot) bool { if mutated == nil || stored.GetDurableSnapshot() == nil || mutated.GetDurableSnapshot() == nil { return false } - if stored.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING || - mutated.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + if stored.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING || + mutated.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { return false } expected := proto.CloneOf(stored) - expected.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + expected.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED return proto.Equal(expected, mutated) } diff --git a/cmd/ateapi/internal/store/atepg/tag_test.go b/cmd/ateapi/internal/store/atepg/tag_test.go index 1b65f7ef0f..0ab2c69815 100644 --- a/cmd/ateapi/internal/store/atepg/tag_test.go +++ b/cmd/ateapi/internal/store/atepg/tag_test.go @@ -40,7 +40,7 @@ func createTestTag(t *testing.T, s *Persistence, tagAtespace, tagName string) *a Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://bucket/atespaces/" + tagAtespace + "/tags/" + tagName}, }, }, diff --git a/cmd/ateapi/internal/store/storecontract/contract.go b/cmd/ateapi/internal/store/storecontract/contract.go index 7ff8970cf5..81838ce7e5 100644 --- a/cmd/ateapi/internal/store/storecontract/contract.go +++ b/cmd/ateapi/internal/store/storecontract/contract.go @@ -1050,7 +1050,7 @@ func runActorContractTests(t *testing.T, setup func(t *testing.T) store.Interfac Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: testActorSnapshotURI("gs://bucket", testAtespace, "snapshot-1")}, }, }}, @@ -1067,7 +1067,7 @@ func runActorContractTests(t *testing.T, setup func(t *testing.T) store.Interfac Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: testActorSnapshotURI("gs://bucket", testAtespace, "snapshot-2")}, }, }}, @@ -1360,7 +1360,7 @@ func newTestSuspendedActor(atespace, name string) *ateapi.Actor { Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: testActorSnapshotURI("gs://private", atespace, name)}, }, }}, @@ -1383,7 +1383,7 @@ func newTestInProgressTag(name string, actor *ateapi.Actor) *ateapi.Tag { Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING, Object: &ateapi.ObjectSnapshot{ SnapshotUri: testTagSnapshotURI("gs://private", atespace, name), }, @@ -1431,8 +1431,8 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) } // A reserved tag has a PENDING snapshot storage entry and is not // usable until the copy lands and finalize marks it COMPLETED. - if got := tag.GetStatus().GetSnapshot().GetDurableSnapshot().GetStatus(); got != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { - t.Errorf("reserved tag snapshot storage status = %v, want PENDING", got) + if got := tag.GetStatus().GetSnapshot().GetDurableSnapshot().GetState(); got != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING { + t.Errorf("reserved tag snapshot storage state = %v, want PENDING", got) } if diff := cmp.Diff(inProgressTag, tag, protocmp.Transform(), ignoreUID, ignoreVersion, ignoreTimestamps); diff != "" { t.Errorf("stored tag mismatch (-want +got):\n%s", diff) @@ -1453,8 +1453,8 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) if gotURI != wantURI { t.Errorf("finalized tag snapshot uri = %q, want %q", gotURI, wantURI) } - if got := ready.GetStatus().GetSnapshot().GetDurableSnapshot().GetStatus(); got != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { - t.Errorf("finalized tag snapshot storage status = %v, want COMPLETED", got) + if got := ready.GetStatus().GetSnapshot().GetDurableSnapshot().GetState(); got != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { + t.Errorf("finalized tag snapshot storage state = %v, want COMPLETED", got) } updated, err := s.UpdateTag(ctx, resources.TagRef{Atespace: "team-a", Name: "production"}, store.PreconditionFrom(ready), func(toUpdate *ateapi.Tag) error { @@ -1789,7 +1789,7 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) // finalizeTag mutates a reserved tag the way the tag workflow's second // transaction does: it marks the pending snapshot storage COMPLETED. func finalizeTag(toUpdate *ateapi.Tag) error { - toUpdate.Status.Snapshot.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + toUpdate.Status.Snapshot.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED return nil } diff --git a/cmd/ateapi/internal/store/storetest/storetest.go b/cmd/ateapi/internal/store/storetest/storetest.go index 7bc219b3d2..2d0f4a5dff 100644 --- a/cmd/ateapi/internal/store/storetest/storetest.go +++ b/cmd/ateapi/internal/store/storetest/storetest.go @@ -101,7 +101,7 @@ func SetupPostgresPersistence(t *testing.T) *atepg.Persistence { func DefaultSystemPrincipals() authz.SystemPrincipals { return authz.SystemPrincipals{ ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEIDs: []string{installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace)}, IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), diff --git a/cmd/ateapi/internal/workerservice/jwt_test.go b/cmd/ateapi/internal/workerservice/jwt_test.go index 355594142a..f84c6939fd 100644 --- a/cmd/ateapi/internal/workerservice/jwt_test.go +++ b/cmd/ateapi/internal/workerservice/jwt_test.go @@ -132,7 +132,7 @@ func pausedStatus(state ateapi.ActorState, node string) *ateapi.ActorStatus { func uploadPendingStatus(node string) *ateapi.ActorStatus { status := pausedStatus(ateapi.ActorState_ACTOR_STATE_SUSPENDED, node) status.Snapshots[0].DurableSnapshot = &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://snaps/root/atespaces/team-a/actors/uid/snap-1"}, } return status @@ -361,7 +361,7 @@ func TestMintWorkerActorJwt_Errors(t *testing.T) { ctx: authed, status: func() *ateapi.ActorStatus { status := uploadPendingStatus("node-1") - status.Snapshots[0].DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + status.Snapshots[0].DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED return status }(), want: codes.OK, diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 7b715e7cc9..2bf57909b2 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -94,7 +94,7 @@ var ( // These SPIFFE IDs hold the matching system component role when they call // over mTLS. controllerSPIFFEID = pflag.String("controller-spiffe-id", installdefaults.ControllerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of ate-controller. Holds the controller authz role, which manages workers.") - egressGatewaySPIFFEID = pflag.String("egress-gateway-spiffe-id", installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-egress. Holds the egress_gateway authz role, which reads and lists actors and reads their egress policies.") + egressGatewaySPIFFEIDs = pflag.StringArray("egress-gateway-spiffe-id", []string{installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv())}, "mTLS SPIFFE IDs of atenet-egress and any other client that holds the egress_gateway authz role, which reads and lists actors and reads their egress policies. Repeat the flag for each ID.") ingressRouterSPIFFEID = pflag.String("ingress-router-spiffe-id", installdefaults.RouterSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-router. Holds the ingress_router authz role, which resumes and lists actors.") apiServerSidecarSPIFFEID = pflag.String("api-server-sidecar-spiffe-id", installdefaults.APIServerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of the ate-api-server pod, which its sidecars share. Holds the api_server_sidecar role, which reads actors so the objectstore control plugin can scope their storage credentials.") @@ -259,7 +259,7 @@ func main() { defer fgaServer.Close() authorizer, policyManager, err := authz.New(shutdownCtx, pool, fgaServer, *authzBootstrapOwners, authz.SystemPrincipals{ ControllerSPIFFEID: *controllerSPIFFEID, - EgressGatewaySPIFFEID: *egressGatewaySPIFFEID, + EgressGatewaySPIFFEIDs: *egressGatewaySPIFFEIDs, IngressRouterSPIFFEID: *ingressRouterSPIFFEID, AteletSPIFFEID: ateletSPIFFEID, APIServerSidecarSPIFFEID: *apiServerSidecarSPIFFEID, @@ -407,6 +407,7 @@ func main() { grpc.ChainUnaryInterceptor(unaryInterceptors...), grpc.ChainStreamInterceptor( apiauthn.StreamServerInterceptor(authCfg), + authz.StreamServerInterceptor(), preview.StreamServerInterceptor, ), ) diff --git a/cmd/atecontroller/internal/controllers/networkpolicy_controller.go b/cmd/atecontroller/internal/controllers/networkpolicy_controller.go index 210539a458..0d67a67d5e 100644 --- a/cmd/atecontroller/internal/controllers/networkpolicy_controller.go +++ b/cmd/atecontroller/internal/controllers/networkpolicy_controller.go @@ -18,11 +18,15 @@ import ( "context" "fmt" + corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" k8errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" + "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -34,6 +38,11 @@ import ( const ( networkPolicyFieldOwner = "ate-networkpolicy" atenetRouterAppName = "atenet-router" + // eventReasonNetworkPolicyConflict is the reason of the Warning event a + // pool emits when checkNetworkPolicyOwner refuses its NetworkPolicy. + eventReasonNetworkPolicyConflict = "NetworkPolicyConflict" + // eventActionApplyNetworkPolicy is the action of that event. + eventActionApplyNetworkPolicy = "ApplyNetworkPolicy" ) type NetworkPolicyReconciler struct { @@ -44,10 +53,15 @@ type NetworkPolicyReconciler struct { // ingress policy admits only that namespace, so a value that does not // match the running router blocks every request to the worker pool. SystemNamespace string + + // Recorder emits a Warning event on the pool when checkNetworkPolicyOwner + // refuses its NetworkPolicy, which leaves the pool without one. + Recorder events.EventRecorder } //+kubebuilder:rbac:groups=ate.dev,resources=workerpools,verbs=get;list;watch //+kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch func (r *NetworkPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) @@ -80,6 +94,19 @@ func (r *NetworkPolicyReconciler) reconcileImpl(ctx context.Context, wp *atev1.W npAC := r.buildNetworkPolicyApplyConfig(wp) + existing := &networkingv1.NetworkPolicy{} + err := r.Get(ctx, types.NamespacedName{Namespace: *npAC.Namespace, Name: *npAC.Name}, existing) + switch { + case k8errors.IsNotFound(err): + case err != nil: + return fmt.Errorf("failed to get NetworkPolicy %s:%s: %w", *npAC.Namespace, *npAC.Name, err) + default: + if err := checkNetworkPolicyOwner(existing, wp); err != nil { + r.Recorder.Eventf(wp, existing, corev1.EventTypeWarning, eventReasonNetworkPolicyConflict, eventActionApplyNetworkPolicy, "%v", err) + return err + } + } + if err := r.Apply(ctx, npAC, client.FieldOwner(networkPolicyFieldOwner), client.ForceOwnership); err != nil { return fmt.Errorf("failed to apply NetworkPolicy %s:%s: %w", *npAC.Namespace, *npAC.Name, err) } @@ -90,6 +117,26 @@ func (r *NetworkPolicyReconciler) reconcileImpl(ctx context.Context, wp *atev1.W return nil } +// checkNetworkPolicyOwner returns an error if np belongs to something other +// than wp. The apply forces ownership, so without this check it would rewrite +// another pool's policy, or a policy someone else created with the same name. +// A policy controlled by wp passes even if its label was edited, so the apply +// restores it. A policy with no controller passes only if it is labeled for wp. +func checkNetworkPolicyOwner(np *networkingv1.NetworkPolicy, wp *atev1.WorkerPool) error { + if owner := metav1.GetControllerOf(np); owner != nil { + if owner.UID == wp.UID { + return nil + } + return fmt.Errorf("refusing to take over NetworkPolicy %s:%s: controlled by %s %s (UID %s)", + np.Namespace, np.Name, owner.Kind, owner.Name, owner.UID) + } + if got := np.Labels[workerPoolLabel]; got != wp.Name { + return fmt.Errorf("refusing to take over NetworkPolicy %s:%s: label %s is %q, want %q", + np.Namespace, np.Name, workerPoolLabel, got, wp.Name) + } + return nil +} + func (r *NetworkPolicyReconciler) buildNetworkPolicyApplyConfig(wp *atev1.WorkerPool) *networkingv1ac.NetworkPolicyApplyConfiguration { np := networkingv1ac.NetworkPolicy(resources.NetworkPolicyName(wp.Name), wp.Namespace). WithLabels(map[string]string{ diff --git a/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go b/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go index 2e07170943..cd43ac2373 100644 --- a/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go +++ b/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go @@ -16,11 +16,18 @@ package controllers import ( "context" + "slices" "testing" networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" + networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" "go.ate.dev/substrate/internal/installdefaults" "go.ate.dev/substrate/internal/resources" ) @@ -112,3 +119,107 @@ func TestBuildNetworkPolicyRelocatedNamespace(t *testing.T) { t.Errorf("ingress namespace selector = %q, want %q", got, relocated) } } + +// TestNetworkPolicyTakeover pre-creates the pool's NetworkPolicy and checks +// that reconcile rewrites it only when it belongs to the pool. +func TestNetworkPolicyTakeover(t *testing.T) { + tests := []struct { + name string + label string + ownerUID types.UID + wantErr bool + }{ + { + name: "labeled for another pool", + label: "other-pool", + wantErr: true, + }, + { + name: "unlabeled", + wantErr: true, + }, + { + name: "controlled by an earlier pool with the same name", + label: "netpol-takeover", + ownerUID: "earlier-uid", + wantErr: true, + }, + { + name: "orphaned policy of this pool", + label: "netpol-takeover", + }, + { + name: "controlled by this pool with an edited label", + label: "edited", + ownerUID: "netpol-takeover-uid", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + // The pool is never created, so only this test reconciles it. + wp := makeWorkerPool("netpol-takeover", "default", 1, "worker:v1") + wp.UID = "netpol-takeover-uid" + + // Apply as the controller's field manager, as an earlier reconcile would. + // The API server rejects a second controller reference written by another + // manager, so the earlier-pool row would pass even without the check. + ac := networkingv1ac.NetworkPolicy(resources.NetworkPolicyName(wp.Name), wp.Namespace). + WithSpec(networkingv1ac.NetworkPolicySpec(). + WithPolicyTypes(networkingv1.PolicyTypeIngress)) + if tt.label != "" { + ac.WithLabels(map[string]string{workerPoolLabel: tt.label}) + } + if tt.ownerUID != "" { + ac.WithOwnerReferences(metav1ac.OwnerReference(). + WithAPIVersion(atev1.GroupVersion.String()). + WithKind("WorkerPool"). + WithName(wp.Name). + WithUID(tt.ownerUID). + WithController(true)) + } + if err := k8sClient.Apply(ctx, ac, client.FieldOwner(networkPolicyFieldOwner)); err != nil { + t.Fatalf("apply NetworkPolicy: %v", err) + } + np := &networkingv1.NetworkPolicy{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: *ac.Name, Namespace: *ac.Namespace}, np); err != nil { + t.Fatalf("get NetworkPolicy: %v", err) + } + deleteOnCleanup(t, np) + + rec := events.NewFakeRecorder(10) + r := &NetworkPolicyReconciler{Client: k8sClient, SystemNamespace: installdefaults.SystemNamespace, Recorder: rec} + err := r.reconcileImpl(ctx, wp) + if (err != nil) != tt.wantErr { + t.Fatalf("reconcileImpl() error = %v, wantErr %v", err, tt.wantErr) + } + var gotEvents, wantEvents []string + for len(rec.Events) > 0 { + gotEvents = append(gotEvents, <-rec.Events) + } + if err != nil { + wantEvents = []string{"Warning " + eventReasonNetworkPolicyConflict + " " + err.Error()} + } + if !slices.Equal(gotEvents, wantEvents) { + t.Errorf("events = %q, want %q", gotEvents, wantEvents) + } + + got := &networkingv1.NetworkPolicy{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(np), got); err != nil { + t.Fatalf("get NetworkPolicy: %v", err) + } + if tt.wantErr { + if got.ResourceVersion != np.ResourceVersion { + t.Errorf("NetworkPolicy was rewritten: %+v", got) + } + return + } + if !metav1.IsControlledBy(got, wp) { + t.Errorf("NetworkPolicy owners = %+v, want controller UID %s", got.OwnerReferences, wp.UID) + } + if got.Labels[workerPoolLabel] != wp.Name { + t.Errorf("NetworkPolicy label %s = %q, want %q", workerPoolLabel, got.Labels[workerPoolLabel], wp.Name) + } + }) + } +} diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index 38395829d6..1213d37b99 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -108,6 +108,7 @@ func TestMain(m *testing.M) { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), SystemNamespace: installdefaults.SystemNamespace, + Recorder: mgr.GetEventRecorder("networkpolicy-controller"), }).SetupWithManager(mgr); err != nil { fmt.Fprintf(os.Stderr, "netpolicy controller setup failed: %v\n", err) os.Exit(1) diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index d264c67226..ec8b240552 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -284,6 +284,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), SystemNamespace: systemNamespace, + Recorder: mgr.GetEventRecorder("networkpolicy-controller"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NetPolicy") os.Exit(1) diff --git a/cmd/atelet/bootid.go b/cmd/atelet/bootid.go index 2f75216fce..715a2a5ecf 100644 --- a/cmd/atelet/bootid.go +++ b/cmd/atelet/bootid.go @@ -15,32 +15,18 @@ package main import ( + "errors" "fmt" + "io/fs" "os" + "path/filepath" "strings" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/atomicfile" + "go.ate.dev/substrate/internal/imagecache" ) -// bootIDPath is where Linux exposes the boot ID: a random UUID generated once -// per boot, so it changes on every reboot, clean or not, and on nothing else. -// It is not namespaced, so atelet reads the node's. -const bootIDPath = "/proc/sys/kernel/random/boot_id" - -// readBootID returns the boot ID read from path. It is never empty, so a local -// snapshot whose manifest records none never matches it. -func readBootID(path string) (string, error) { - b, err := os.ReadFile(path) - if err != nil { - return "", fmt.Errorf("while reading the boot ID: %w", err) - } - id := strings.TrimSpace(string(b)) - if id == "" { - return "", fmt.Errorf("boot ID file %s is empty", path) - } - return id, nil -} - // localSnapshotBootCheck returns the snapshot tracker's check of whether an // actor's local snapshot was written on the boot bootID: whether the // snapshot's manifest records bootID. A manifest that is missing or does not @@ -55,3 +41,75 @@ func localSnapshotBootCheck(bootID string) func(actorUID, snapshotUUID string) b return err == nil && rec.BootID == bootID } } + +// clearCachesAfterReboot empties each of cacheDirs, and removes every actor's +// OCI bundle dir and every worker pod's default-runtime dir, when markerPath +// records a boot other than bootID. Then it records bootID there. It reports +// whether it cleared anything. +// +// Cache writes aren't synced, so an unclean reboot can leave cache entries +// and bundle specs empty or truncated. The bundle and default-runtime dirs +// hold specs naming cached layers and hard links to cached assets, which +// atelet reuses instead of linking again. No sandbox survives a reboot, so +// nothing still uses any of them. Run recreates an actor's bundle dir, and the +// worker sandbox syncer recreates a worker's default-runtime dir. +// +// A missing marker only records bootID. atelet may be starting with this +// check for the first time while actors run on what it would clear. +func clearCachesAfterReboot(bootID, markerPath, actorsDir, workersDir string, cacheDirs ...string) (bool, error) { + b, err := os.ReadFile(markerPath) + switch { + case errors.Is(err, fs.ErrNotExist): + return false, writeCacheBootID(markerPath, bootID) + case err != nil: + return false, fmt.Errorf("while reading the cache boot ID: %w", err) + case strings.TrimSpace(string(b)) == bootID: + return false, nil + } + for _, dir := range cacheDirs { + if err := removeChildren(dir, ""); err != nil { + return false, err + } + } + if err := removeChildren(actorsDir, "bundles"); err != nil { + return false, err + } + if err := removeChildren(workersDir, defaultRuntimeDirsName); err != nil { + return false, err + } + return true, writeCacheBootID(markerPath, bootID) +} + +// removeChildren removes the named entry in each child directory of dir, or +// each child itself when name is empty. A missing dir has nothing to remove. +func removeChildren(dir, name string) error { + if dir == "" { + return nil + } + children, err := os.ReadDir(dir) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("while listing %s: %w", dir, err) + } + for _, child := range children { + if name != "" && !child.IsDir() { + continue + } + path := filepath.Join(dir, child.Name(), name) + // RemoveAllWritable, not os.RemoveAll: unpacked image layers keep + // the image's directory modes, which can be read-only. + if err := imagecache.RemoveAllWritable(path); err != nil { + return fmt.Errorf("while removing %s: %w", path, err) + } + } + return nil +} + +func writeCacheBootID(path, bootID string) error { + if err := atomicfile.WriteSync(path, []byte(bootID+"\n"), 0o600); err != nil { + return fmt.Errorf("while recording the cache boot ID: %w", err) + } + return nil +} diff --git a/cmd/atelet/bootid_test.go b/cmd/atelet/bootid_test.go index 52046be46b..2ef67806aa 100644 --- a/cmd/atelet/bootid_test.go +++ b/cmd/atelet/bootid_test.go @@ -15,6 +15,8 @@ package main import ( + "errors" + "io/fs" "os" "path/filepath" "testing" @@ -22,36 +24,6 @@ import ( "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" ) -func TestReadBootID(t *testing.T) { - t.Run("reads and trims boot ID", func(t *testing.T) { - p := filepath.Join(t.TempDir(), "boot_id") - const want = "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f" - if err := os.WriteFile(p, []byte(want+"\n"), 0o600); err != nil { - t.Fatal(err) - } - got, err := readBootID(p) - if err != nil || got != want { - t.Errorf("readBootID = (%q, %v), want (%q, nil)", got, err, want) - } - }) - - t.Run("rejects empty file", func(t *testing.T) { - p := filepath.Join(t.TempDir(), "boot_id") - if err := os.WriteFile(p, []byte(" \n\t"), 0o600); err != nil { - t.Fatal(err) - } - if _, err := readBootID(p); err == nil { - t.Error("readBootID accepted an empty boot ID file") - } - }) - - t.Run("rejects missing file", func(t *testing.T) { - if _, err := readBootID(filepath.Join(t.TempDir(), "missing")); err == nil { - t.Error("readBootID accepted a missing file") - } - }) -} - // localSnapshotBootCheck matches a local snapshot only when its manifest // records the given boot ID. func TestLocalSnapshotBootCheck(t *testing.T) { @@ -89,3 +61,87 @@ func TestLocalSnapshotBootCheck(t *testing.T) { }) } } + +// clearCachesAfterReboot clears the caches, the actors' bundle dirs and the +// workers' default-runtime dirs only once the node has rebooted since the +// recorded boot, and leaves everything else in place. +func TestClearCachesAfterReboot(t *testing.T) { + for _, tc := range []struct { + name string + // marker is the recorded boot ID, or empty for no marker file. + marker string + wantCleared bool + }{ + {name: "rebooted since the recorded boot", marker: "boot-1\n", wantCleared: true}, + {name: "same boot", marker: "boot-2\n"}, + {name: "no recorded boot", marker: ""}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + marker := filepath.Join(root, "cache-boot-id") + imageCache := filepath.Join(root, "image-cache") + assetCache := filepath.Join(root, "sandbox-asset-cache") + actors := filepath.Join(root, "actors") + workers := filepath.Join(root, "workers") + + cleared := []string{ + filepath.Join(imageCache, "layers", "sha256", "abc", "fs", "etc", "ca.crt"), + filepath.Join(imageCache, "version"), + filepath.Join(assetCache, "entries", "k1", "data"), + filepath.Join(actors, "a1", "bundles", "c1", "overlay.json"), + filepath.Join(workers, "w1", "default-runtime", "sha256-x", "runsc"), + } + kept := []string{ + filepath.Join(actors, "a1", "snapshot-state.json"), + filepath.Join(actors, "a1", "sandbox-assets.json"), + filepath.Join(workers, "w1", "sandbox-runtimes.json"), + filepath.Join(root, "snapshots", "a1", "local-checkpoint"), + } + for _, p := range append(append([]string{}, cleared...), kept...) { + if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, nil, 0o600); err != nil { + t.Fatal(err) + } + } + // A plain file directly in the actors dir has no bundle dir. + if err := os.WriteFile(filepath.Join(actors, "stray"), nil, 0o600); err != nil { + t.Fatal(err) + } + // Unpacked layers keep the image's directory modes. + readOnly := filepath.Join(imageCache, "layers", "sha256", "abc", "fs", "etc") + if err := os.Chmod(readOnly, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(readOnly, 0o700) }) + if tc.marker != "" { + if err := os.WriteFile(marker, []byte(tc.marker), 0o600); err != nil { + t.Fatal(err) + } + } + + got, err := clearCachesAfterReboot("boot-2", marker, actors, workers, imageCache, assetCache, "") + if err != nil { + t.Fatalf("clearCachesAfterReboot: %v", err) + } + if got != tc.wantCleared { + t.Errorf("clearCachesAfterReboot cleared = %v, want %v", got, tc.wantCleared) + } + for _, p := range cleared { + _, err := os.Stat(p) + if gone := errors.Is(err, fs.ErrNotExist); gone != tc.wantCleared { + t.Errorf("%s removed = %v, want %v (stat err %v)", p, gone, tc.wantCleared, err) + } + } + for _, p := range append(kept, imageCache, assetCache, filepath.Join(actors, "stray")) { + if _, err := os.Stat(p); err != nil { + t.Errorf("%s should stay: %v", p, err) + } + } + if b, err := os.ReadFile(marker); err != nil || string(b) != "boot-2\n" { + t.Errorf("marker = (%q, %v), want (%q, nil)", b, err, "boot-2\n") + } + }) + } +} diff --git a/cmd/atelet/internal/apivalidation/zz_generated.validation.go b/cmd/atelet/internal/apivalidation/zz_generated.validation.go index a312fc66a6..8b6884598a 100644 --- a/cmd/atelet/internal/apivalidation/zz_generated.validation.go +++ b/cmd/atelet/internal/apivalidation/zz_generated.validation.go @@ -3152,6 +3152,83 @@ func Validate_RunRequest( return errs } +// Validate_RuntimeOverhead validates an instance of RuntimeOverhead according +// to declarative validation rules in the API schema. +func Validate_RuntimeOverhead( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateletpb.RuntimeOverhead) (errs field.ErrorList) { + + { // field ateletpb.RuntimeOverhead.MemoryBytes + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 1099511627776); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.RuntimeOverhead) *int64 { + return &oldObj.MemoryBytes + }) + errs = append(errs, fn(fldPath.Child("memory_bytes"), &obj.MemoryBytes, oldVal, oldObj != nil)...) + } + + { // field ateletpb.RuntimeOverhead.MemoryBytesPerGib + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 1073741824); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.RuntimeOverhead) *int64 { + return &oldObj.MemoryBytesPerGib + }) + errs = append(errs, fn(fldPath.Child("memory_bytes_per_gib"), &obj.MemoryBytesPerGib, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_SecurityContext validates an instance of SecurityContext according // to declarative validation rules in the API schema. func Validate_SecurityContext( @@ -4624,6 +4701,36 @@ func Validate_WorkerSandboxRuntime( errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) } + { // field ateletpb.WorkerSandboxRuntime.ActorOverhead + fn := func( + fldPath *field.Path, + obj, oldObj *ateletpb.RuntimeOverhead, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_RuntimeOverhead(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.WorkerSandboxRuntime) *ateletpb.RuntimeOverhead { + return oldObj.ActorOverhead + }) + errs = append(errs, fn(fldPath.Child("actor_overhead"), obj.ActorOverhead, oldVal, oldObj != nil)...) + } + return errs } diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index 67bab746c1..6f7ee9b015 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -57,6 +57,11 @@ var ( // again. SandboxRuntimeIndexFile = filepath.Join(nodepath.BasePath, "sandbox-runtime-index.json") + // CacheBootIDFile records the boot ID of the boot that last wrote atelet's + // caches. atelet clears them at startup when the node has rebooted since + // (see clearCachesAfterReboot in package main). + CacheBootIDFile = filepath.Join(nodepath.BasePath, "cache-boot-id") + // SnapshotCacheDir is the node-local cache of shared snapshot files, the // snapshots many actors are created from (see // cmd/atelet/internal/filecache). It lives under BasePath because cache @@ -75,8 +80,8 @@ func ActorPath(actorUID string) string { } // ActorSnapshotPath is the actor's snapshot staging directory, which may be on -// a different filesystem from ActorPath. Checkpoints rename files between its -// subdirectories, so they must stay on one filesystem. +// a different filesystem from ActorPath. Checkpoints link and rename files +// between its subdirectories, so they must stay on one filesystem. func ActorSnapshotPath(actorUID string) string { return filepath.Join( nodepath.SnapshotsDir, diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 1d96035c65..e51bdf6444 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -74,7 +74,8 @@ func useTempNodeDirs(t *testing.T) { // the request, never derived from the actor UID. type fakeWorker struct { ateworkerpb.UnimplementedWorkerServer - // mu guards the record maps: concurrent RPCs record at the same time. + // mu guards the record maps and stoppedByResp: concurrent RPCs record at + // the same time. mu sync.Mutex // snapshotFiles are written at checkpoint and reported back to atelet as // the exact set the snapshot consists of. @@ -85,6 +86,9 @@ type fakeWorker struct { // restoreErr, if set, fails every RestoreWorkload after it recorded the // request, as a worker that could not start the sandbox would. restoreErr error + // checkpointErr, if set, fails every CheckpointWorkload after it recorded + // the request, before it writes any snapshot file. + checkpointErr error // actorDirs records the ActorDirs each RPC arrived with, by RPC name. actorDirs map[string]*ateworkerpb.ActorDirs // preserveRestoreDir records the PreserveRestoreDir flag from the most @@ -104,10 +108,34 @@ type fakeWorker struct { // onRestore, if non-nil, runs inside RestoreWorkload after it recorded // the request, before it fails with restoreErr or succeeds. onRestore func() + // stopsWorkload, if set, makes a checkpoint stop the workload, as a real + // worker's does. Until a Run or Restore starts it again, CheckpointWorkload + // answers with the stopping checkpoint's response while the checkpoint dir + // still holds its files, and fails otherwise. + stopsWorkload bool + // stoppedByResp is the response of the checkpoint that stopped the + // workload, nil while the workload runs. Guarded by mu. + stoppedByResp *ateworkerpb.CheckpointWorkloadResponse // terminations counts TerminateWorkload calls. terminations atomic.Int32 } +// setStoppedBy records the response of the checkpoint that stopped the +// workload, or nil once the workload runs again. +func (f *fakeWorker) setStoppedBy(resp *ateworkerpb.CheckpointWorkloadResponse) { + f.mu.Lock() + defer f.mu.Unlock() + f.stoppedByResp = resp +} + +// stoppedBy returns the response of the checkpoint that stopped the +// workload, or nil while the workload runs. +func (f *fakeWorker) stoppedBy() *ateworkerpb.CheckpointWorkloadResponse { + f.mu.Lock() + defer f.mu.Unlock() + return f.stoppedByResp +} + func (f *fakeWorker) recordFidelity(rpc string, fidelity ateworkerpb.SnapshotFidelity) { f.mu.Lock() defer f.mu.Unlock() @@ -128,13 +156,25 @@ func (f *fakeWorker) recordActorDirs(rpc string, actorDirs *ateworkerpb.ActorDir func (f *fakeWorker) RunWorkload(_ context.Context, req *ateworkerpb.RunWorkloadRequest) (*ateworkerpb.RunWorkloadResponse, error) { f.recordActorDirs("RunWorkload", req.GetActorDirs()) + f.setStoppedBy(nil) return &ateworkerpb.RunWorkloadResponse{WorkerEpoch: f.workerEpoch}, nil } func (f *fakeWorker) CheckpointWorkload(ctx context.Context, req *ateworkerpb.CheckpointWorkloadRequest) (*ateworkerpb.CheckpointWorkloadResponse, error) { f.recordActorDirs("CheckpointWorkload", req.GetActorDirs()) f.recordFidelity("CheckpointWorkload", req.GetFidelity()) + if f.checkpointErr != nil { + return nil, f.checkpointErr + } dir := req.GetActorDirs().GetCheckpointDir() + if resp := f.stoppedBy(); resp != nil { + for _, name := range resp.GetSnapshotFiles() { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + return nil, status.Errorf(codes.Internal, "no workload to checkpoint: %v", err) + } + } + return resp, nil + } names := make([]string, 0, len(f.snapshotFiles)) for name, body := range f.snapshotFiles { if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { @@ -145,10 +185,14 @@ func (f *fakeWorker) CheckpointWorkload(ctx context.Context, req *ateworkerpb.Ch if f.onCheckpoint != nil { f.onCheckpoint() } + resp := &ateworkerpb.CheckpointWorkloadResponse{SnapshotFiles: names} + if f.stopsWorkload { + f.setStoppedBy(resp) + } if err := ctx.Err(); err != nil { return nil, err } - return &ateworkerpb.CheckpointWorkloadResponse{SnapshotFiles: names}, nil + return resp, nil } func (f *fakeWorker) RestoreWorkload(_ context.Context, req *ateworkerpb.RestoreWorkloadRequest) (*ateworkerpb.RestoreWorkloadResponse, error) { @@ -161,6 +205,7 @@ func (f *fakeWorker) RestoreWorkload(_ context.Context, req *ateworkerpb.Restore if f.restoreErr != nil { return nil, f.restoreErr } + f.setStoppedBy(nil) dir := req.GetActorDirs().GetRestoreDir() f.restored = map[string]string{} for name := range f.snapshotFiles { @@ -1479,9 +1524,10 @@ func (a *pluginActor) snapshot(name string, generation int32) *ateletpb.Snapshot return lifecycleSnapshot(pluginActorUID, name, generation, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) } -func (a *pluginActor) checkpoint(t *testing.T, snap *ateletpb.Snapshot, mode ateletpb.UploadMode) (*ateletpb.CheckpointResponse, error) { - t.Helper() - return a.s.Checkpoint(t.Context(), &ateletpb.CheckpointRequest{ +// checkpointRequest is the CheckpointRequest the control plane sends to take +// snap under mode. +func (a *pluginActor) checkpointRequest(snap *ateletpb.Snapshot, mode ateletpb.UploadMode) *ateletpb.CheckpointRequest { + return &ateletpb.CheckpointRequest{ Atespace: "ate-demo", ActorName: "counter", ActorUid: pluginActorUID, @@ -1491,7 +1537,12 @@ func (a *pluginActor) checkpoint(t *testing.T, snap *ateletpb.Snapshot, mode ate Spec: a.spec, Snapshot: snap, UploadConfig: &ateletpb.UploadConfig{Mode: mode}, - }) + } +} + +func (a *pluginActor) checkpoint(t *testing.T, snap *ateletpb.Snapshot, mode ateletpb.UploadMode) (*ateletpb.CheckpointResponse, error) { + t.Helper() + return a.s.Checkpoint(t.Context(), a.checkpointRequest(snap, mode)) } // restoreLocal resumes the actor from its local snapshot named name, taken at @@ -1784,6 +1835,62 @@ func TestRetriedCheckpointAnswersFromCommittedSnapshot(t *testing.T) { } } +// TestRetriedCheckpointAfterUncommittedStaging retries a deferred Checkpoint +// whose first attempt died after the worker stopped the workload and atelet +// began staging the local snapshot, but before the snapshot committed. The +// checkpoint dir still holds the worker's files, so the worker answers the +// retry from them, and the retry commits the snapshot. +func TestRetriedCheckpointAfterUncommittedStaging(t *testing.T) { + for _, tc := range []struct { + name string + // notWritten names snapshot files the worker did not write. Staging + // stops at the first of them, as it would if atelet died there. + notWritten []string + }{ + {name: "staging interrupted", notWritten: []string{"pages.img"}}, + {name: "local snapshot published"}, + } { + t.Run(tc.name, func(t *testing.T) { + a := runPluginActor(t) + a.worker.stopsWorkload = true + var captures atomic.Int32 + a.worker.onCheckpoint = func() { captures.Add(1) } + snap := a.snapshot("snap-1", 1) + req := a.checkpointRequest(snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + + // The first attempt gets as far as staging the local snapshot. + wresp, err := a.worker.CheckpointWorkload(t.Context(), &ateworkerpb.CheckpointWorkloadRequest{ + ActorDirs: ateletpath.ActorDirs(pluginActorUID), + Fidelity: toWorkerFidelity(snap.GetFidelity()), + }) + if err != nil { + t.Fatalf("CheckpointWorkload: %v", err) + } + rec, err := readSandboxRecord(pluginActorUID) + if err != nil { + t.Fatal(err) + } + rec.SnapshotFiles = slices.Concat(wresp.GetSnapshotFiles(), tc.notWritten) + err = a.s.moveLocalCheckpoint(t.Context(), req, rec) + if wantErr := len(tc.notWritten) > 0; (err != nil) != wantErr { + t.Fatalf("moveLocalCheckpoint = %v, want error %t", err, wantErr) + } + + resp, err := a.s.Checkpoint(t.Context(), req) + if err != nil { + t.Fatalf("retried Checkpoint: %v", err) + } + if got := resp.GetState().GetLatestLocal().GetSnapshot(); !proto.Equal(got, snap) { + t.Errorf("latest_local = %v, want %v", got, snap) + } + if n := captures.Load(); n != 1 { + t.Errorf("worker captured the workload %d times, want once", n) + } + a.restoreLocal(t, "snap-1", 1) + }) + } +} + // TestTerminateWaitsForInFlightRPC checks that Terminate waits on the // per-actor lock while Checkpoint or PromoteSnapshot is in flight, then cleans // up the actor's tracked state and local files once that RPC returns. @@ -1912,17 +2019,6 @@ func TestFailedCheckpointKeepsLocalSnapshot(t *testing.T) { a.store.putErr = errors.New("object store unavailable") }, }, - { - name: "pause move fails", - snap: "pause-snap-2", - mode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, - fail: func(t *testing.T, a *pluginActor) { - // A file where the new snapshot's directory belongs fails the move. - if err := os.WriteFile(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-2"), nil, 0o600); err != nil { - t.Fatal(err) - } - }, - }, { name: "missing snapshot file", snap: "pause-snap-2", @@ -3871,3 +3967,27 @@ func TestCheckpointFinishesWhenCallerCancels(t *testing.T) { unlock() }) } + +// A worker that is unreachable when Checkpoint calls it fails the checkpoint +// with Unavailable, however many times in a row, so ateapi keeps the actor +// SUSPENDING for a retry instead of crashing an actor whose state only the +// worker holds. The retry checkpoints once the worker answers again. +func TestCheckpointKeepsWorkerUnavailable(t *testing.T) { + a := runPluginActor(t) + snap := a.snapshot("pause-snap-1", 1) + checkpointCode := func() codes.Code { + _, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + return apierror.Code(err) + } + + a.worker.checkpointErr = status.Error(codes.Unavailable, `connection error: desc = "transport: Error while dialing: dial unix /var/lib/ate/worker/sock: connect: no such file or directory"`) + for i := 1; i <= 5; i++ { + if got := checkpointCode(); got != codes.Unavailable { + t.Fatalf("Checkpoint %d with the worker unreachable: code = %v, want %v", i, got, codes.Unavailable) + } + } + a.worker.checkpointErr = nil + if got := checkpointCode(); got != codes.OK { + t.Fatalf("Checkpoint with the worker back: code = %v, want %v", got, codes.OK) + } +} diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index fb145da374..1987f2bae9 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -52,6 +52,7 @@ import ( "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/atelet" + "go.ate.dev/substrate/internal/bootid" "go.ate.dev/substrate/internal/clustertrustbundle" "go.ate.dev/substrate/internal/credbundle" "go.ate.dev/substrate/internal/imagecache" @@ -85,10 +86,14 @@ import ( "google.golang.org/grpc/status" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/events" "k8s.io/utils/clock" "k8s.io/utils/lru" ) @@ -250,9 +255,30 @@ func main() { imageCredsKeychain = kc } + // Validated before the caches are cleared, so a directory the flags + // reject is never emptied. if err := validateImageCacheGCFlags(); err != nil { serverboot.Fatal(ctx, "Invalid image cache GC flags", err) } + if err := validateSandboxAssetCacheFlags(); err != nil { + serverboot.Fatal(ctx, "Invalid sandbox asset cache flags", err) + } + if err := validateSnapshotCacheFlags(); err != nil { + serverboot.Fatal(ctx, "Invalid snapshot cache flags", err) + } + bootID, err := bootid.Read(bootid.Path) + if err != nil { + serverboot.Fatal(ctx, "Failed to read the node's boot ID", err) + } + cleared, err := clearCachesAfterReboot(bootID, ateletpath.CacheBootIDFile, nodepath.ActorsDir, nodepath.WorkersDir(), + *imageCacheDir, *sandboxAssetCacheDir, *snapshotCacheDir) + if err != nil { + serverboot.Fatal(ctx, "Failed to clear the caches after a reboot", err) + } + if cleared { + slog.InfoContext(ctx, "Cleared the node-local caches after a node reboot") + } + imageCache, err := imagecache.New(*imageCacheDir, imagecache.WithKeychain(imageCredsKeychain), imagecache.WithLocalhostRegistryReplacement(*localhostRegistryReplacement), @@ -267,9 +293,6 @@ func main() { go newImageCacheGC(imageCache, *imageCacheDir).Run(ctx) } - if err := validateSandboxAssetCacheFlags(); err != nil { - serverboot.Fatal(ctx, "Invalid sandbox asset cache flags", err) - } sandboxAssetCache, err := openSandboxAssetCache(ctx, *sandboxAssetCacheDir, *sandboxAssetCacheMinAge, *sandboxAssetFetchTimeout, nodepath.ActorsDir, nodepath.WorkersDir()) if err != nil { serverboot.Fatal(ctx, "Failed to open sandbox asset cache", err) @@ -282,9 +305,6 @@ func main() { if err := filecachegc.Start(ctx, ateattr.FileCacheSandboxAsset, sandboxAssetCache, sandboxAssetCacheGCFlags, fileCacheInstruments); err != nil { serverboot.Fatal(ctx, "Failed to start the sandbox asset cache GC", err) } - if err := validateSnapshotCacheFlags(); err != nil { - serverboot.Fatal(ctx, "Invalid snapshot cache flags", err) - } snapshotCache, err := openSnapshotCache(ctx, *snapshotCacheDir, *snapshotCacheMinAge, *snapshotCacheFetchTimeout) if err != nil { serverboot.Fatal(ctx, "Failed to open snapshot cache", err) @@ -387,10 +407,6 @@ func main() { reporter = &trackerReporter{workers: workerService} } - bootID, err := readBootID(bootIDPath) - if err != nil { - serverboot.Fatal(ctx, "Failed to read the node's boot ID", err) - } // The snapshot tracker's workers get a context of their own, cancelled only // once the gRPC server has drained, so background snapshot work keeps // running while in-flight RPCs finish. @@ -526,6 +542,12 @@ func main() { // still covers correctness. prewarmer := newRuntimePrewarmer(runtimes, wmService) go prewarmer.run(ctx) + eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: k8sClient.EventsV1()}) + if err := eventBroadcaster.StartRecordingToSinkWithContext(ctx); err != nil { + serverboot.Fatal(ctx, "Failed to start the event broadcaster", err) + } + defer eventBroadcaster.Shutdown() + // Keep a sandbox runtimes file in each local worker's directory: the // runtimes of the enabled versions of its pool's SandboxConfig, // rewritten when the config changes. @@ -534,7 +556,8 @@ func main() { ateFactory.Ate().V1().WorkerPools(), ateFactory.Ate().V1().SandboxConfigs(), wmService, - nodepath.WorkersDir(), runtimes, prewarmer) + nodepath.WorkersDir(), runtimes, prewarmer, + eventBroadcaster.NewRecorder(newEventScheme(), "atelet")) if err != nil { // Without the syncer no runtime is ever indexed, so no actor could run. serverboot.Fatal(ctx, "Failed to start the worker sandbox runtimes syncer", err) @@ -573,6 +596,16 @@ func main() { slog.InfoContext(ctx, "Shutdown complete") } +// newEventScheme returns a Scheme with core Kubernetes and ATE CRD types +// registered so the event recorder can resolve ObjectReferences for both +// regarding (Pod) and related (SandboxConfig, WorkerPool). +func newEventScheme() *k8sruntime.Scheme { + s := k8sruntime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(s)) + utilruntime.Must(atev1.AddToScheme(s)) + return s +} + // servingCheck is the readiness check that holds /readyz at 503 until served // is closed, which main does once the gRPC port is open. An atelet that // stops before then never closes it, and so is never ready. @@ -750,7 +783,7 @@ type Atelet struct { // failures, so a restore that can never succeed stops being retried (see // boundRetryableRestoreError). restoreFailures restoreFailureCounter - // bootID is the node's boot ID (see readBootID), recorded in the manifest + // bootID is the node's boot ID (see bootid.Read), recorded in the manifest // of each local snapshot. A local snapshot recording another, or none, // was written before the node last rebooted. bootID string @@ -1108,9 +1141,9 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest }) dWorker = time.Since(tWorker) if err != nil { - // TODO: Worker should classify checkpoint failures, and set "should-crash" - // in the metadata if the error is not retriable. - return nil, fmt.Errorf("while calling worker CheckpointWorkload: %w", err) + // TODO(#3046): Worker should classify checkpoint failures that left + // the workload intact as retriable. + return nil, workerCallError("CheckpointWorkload", err) } // The snapshot commits on this node once worker has written it, in either // upload mode. @@ -1300,7 +1333,9 @@ func (s *Atelet) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Checkpoi // Stage the files worker reported and the snapshot manifest inside // checkpointDir before renaming the directory into local-checkpoint/, so a // failure leaves nothing under local-checkpoint/ and ResetActorDirs cleans - // checkpointDir up. + // checkpointDir up. Staging links the files, so checkpointDir still holds + // them for a retried CheckpointWorkload if atelet dies before the snapshot + // commits. stageDir := filepath.Join(checkpointDir, ".staged") if err := root.RemoveAll(stageDir); err != nil { return fmt.Errorf("while clearing local checkpoint staging directory: %w", err) @@ -1324,8 +1359,8 @@ func (s *Atelet) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Checkpoi if err := root.MkdirAll(filepath.Dir(dst), 0o700); err != nil { return fmt.Errorf("while creating parent directory for %s: %w", dst, err) } - if err := root.Rename(src, dst); err != nil { - return fmt.Errorf("failed to move %s to %s: %w", src, dst, err) + if err := root.Link(src, dst); err != nil { + return fmt.Errorf("failed to link %s to %s: %w", src, dst, err) } } @@ -1344,6 +1379,11 @@ func (s *Atelet) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Checkpoi if err := root.MkdirAll(filepath.Dir(localDir), 0o700); err != nil { return fmt.Errorf("while creating local checkpoint directory: %w", err) } + // An attempt that died after this rename, before the snapshot committed, + // left localDir behind. + if err := root.RemoveAll(localDir); err != nil { + return fmt.Errorf("while removing uncommitted local snapshot: %w", err) + } if err := root.Rename(stageDir, localDir); err != nil { return fmt.Errorf("failed to move %s to %s: %w", stageDir, localDir, err) } @@ -1874,9 +1914,12 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res } // workerCallError is atelet's error for a failed call to the worker's rpc. A -// worker that is unavailable, refusing workloads until it registers or gone -// with a restart, is reported as Unavailable, so the caller retries once the -// worker process has registered. Anything else reaches the caller as Internal. +// worker that is unavailable, refusing workloads until it registers or +// unreachable because its process is gone, is reported as Unavailable, so +// ateapi keeps the actor's state and assignment for a retry instead of +// crashing it. Once a restarted worker registers again, or when its pod is +// gone, ateapi crashes the actors it hosted on its own. Anything else reaches +// the caller as Internal. func workerCallError(rpc string, err error) error { if status.Code(err) == codes.Unavailable { return apierror.Unavailable("worker %s: the worker is unavailable: %s", rpc, status.Convert(err).Message()) diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 4da514ea30..d3daf4e277 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -1657,7 +1657,7 @@ func TestServingCheck(t *testing.T) { // A worker that is unavailable, whether refusing workloads until it registers // or unreachable after a restart, reaches atelet's caller as Unavailable so the -// resume is retried. Any other worker failure is Internal. +// resume or suspend is retried. Any other worker failure is Internal. func TestWorkerCallError(t *testing.T) { for _, tc := range []struct { name string @@ -1667,6 +1667,8 @@ func TestWorkerCallError(t *testing.T) { {name: "worker refuses", err: status.Error(codes.Unavailable, "worker has not registered yet"), want: codes.Unavailable}, {name: "worker unreachable", err: fmt.Errorf("dial: %w", status.Error(codes.Unavailable, "connection refused")), want: codes.Unavailable}, {name: "worker fails", err: status.Error(codes.Internal, "sandbox would not start"), want: codes.Internal}, + {name: "worker canceled", err: status.Error(codes.Canceled, "gave up waiting for the actor's lock: context canceled"), want: codes.Internal}, + {name: "worker deadline exceeded", err: status.Error(codes.DeadlineExceeded, "gave up waiting for the actor's lock: context deadline exceeded"), want: codes.Internal}, {name: "plain error", err: errors.New("boom"), want: codes.Internal}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/atelet/runtime_prewarm_test.go b/cmd/atelet/runtime_prewarm_test.go index 811c97e817..8c8d1778bb 100644 --- a/cmd/atelet/runtime_prewarm_test.go +++ b/cmd/atelet/runtime_prewarm_test.go @@ -34,6 +34,7 @@ import ( k8sruntime "k8s.io/apimachinery/pkg/runtime" kubeinformers "k8s.io/client-go/informers" kubefake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/events" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" @@ -316,7 +317,7 @@ func TestSyncPodPrewarmsDefaultRuntime(t *testing.T) { if err := os.MkdirAll(stale, 0o700); err != nil { t.Fatal(err) } - if err := s.syncPod(context.Background(), pod); err != nil { + if _, err := s.syncPod(context.Background(), pod); err != nil { t.Fatalf("syncPod: %v", err) } wr, ok := s.index.worker("pod-a") @@ -344,7 +345,7 @@ func TestSyncPodPrewarmsDefaultRuntime(t *testing.T) { if err := os.WriteFile(marker, nil, 0o600); err != nil { t.Fatal(err) } - if err := s.syncPod(context.Background(), pod); err != nil { + if _, err := s.syncPod(context.Background(), pod); err != nil { t.Fatalf("syncPod again: %v", err) } if got := rec.items(); len(got) != 2 || got[1] != want[0] { @@ -370,7 +371,7 @@ func TestSyncPodPrewarmsDefaultRuntime(t *testing.T) { if err := os.MkdirAll(previous, 0o700); err != nil { t.Fatal(err) } - if err := s.syncPod(context.Background(), pod); err == nil { + if _, err := s.syncPod(context.Background(), pod); err == nil { t.Fatal("syncPod with the default's probe failing = nil, want an error") } if got := rec.items(); len(got) != 0 { @@ -459,7 +460,7 @@ func TestDefaultRuntimePrewarmThroughSyncer(t *testing.T) { ateFactory.Ate().V1().WorkerPools(), ateFactory.Ate().V1().SandboxConfigs(), newScriptAssetFetcher(t), // the syncer's runsc probe; prewarm goes through herder - workersDir, index, p) + workersDir, index, p, &events.FakeRecorder{}) if err != nil { t.Fatalf("startWorkerSandboxSyncer: %v", err) } diff --git a/cmd/atelet/worker_actor_jwt.go b/cmd/atelet/worker_actor_jwt.go index d4290b2063..6cbe6adec0 100644 --- a/cmd/atelet/worker_actor_jwt.go +++ b/cmd/atelet/worker_actor_jwt.go @@ -23,7 +23,7 @@ import ( "k8s.io/utils/lru" "go.ate.dev/substrate/api/ateapi/v1" - "go.ate.dev/substrate/internal/objectstoreplugin/exchange" + "go.ate.dev/substrate/internal/objectstoreplugin/actorauthz" ) // actorRef identifies the actor a transfer is for, as atelet's RPCs carry it. @@ -78,7 +78,7 @@ func (j *workerActorJWTs) get(ctx context.Context, actor actorRef) (string, erro resp, err := j.workers.MintWorkerActorJwt(ctx, &ateapi.MintWorkerActorJwtRequest{ Actor: &ateapi.ObjectRef{Atespace: actor.atespace, Name: actor.name}, ActorUid: actor.uid, - Audiences: []string{exchange.ActorJWTAudience}, + Audiences: []string{actorauthz.Audience}, }) switch status.Code(err) { case codes.OK: diff --git a/cmd/atelet/worker_actor_jwt_test.go b/cmd/atelet/worker_actor_jwt_test.go index e646143c03..6b842b206b 100644 --- a/cmd/atelet/worker_actor_jwt_test.go +++ b/cmd/atelet/worker_actor_jwt_test.go @@ -27,7 +27,7 @@ import ( "go.ate.dev/substrate/api/ateapi/v1" "go.ate.dev/substrate/api/objectstore/v1" - "go.ate.dev/substrate/internal/objectstoreplugin/exchange" + "go.ate.dev/substrate/internal/objectstoreplugin/actorauthz" ) // testActor is the actor the transfer tests act for. @@ -81,7 +81,7 @@ func TestWorkerActorJWTs_MintsOncePerActorUntilNearExpiry(t *testing.T) { if req.GetActor().GetAtespace() != testActor.atespace || req.GetActor().GetName() != testActor.name || req.GetActorUid() != testActor.uid { t.Errorf("mint request names %v uid=%s", req.GetActor(), req.GetActorUid()) } - if len(req.GetAudiences()) != 1 || req.GetAudiences()[0] != exchange.ActorJWTAudience { + if len(req.GetAudiences()) != 1 || req.GetAudiences()[0] != actorauthz.Audience { t.Errorf("mint audiences = %v, want the exchange's", req.GetAudiences()) } diff --git a/cmd/atelet/worker_sandbox_syncer.go b/cmd/atelet/worker_sandbox_syncer.go index 45a82a7488..68ae5edc50 100644 --- a/cmd/atelet/worker_sandbox_syncer.go +++ b/cmd/atelet/worker_sandbox_syncer.go @@ -22,7 +22,8 @@ package main // - process: for each UID, sync the pod, or remove its worker directory // once the pod is gone // - finish: forget the UID, or retry it with backoff, capped at -// syncMaxRetries while the pool's SandboxConfig is unresolved +// syncMaxRetries while the pool's SandboxConfig is unresolved, and +// emit a Warning Event on the pod when a sync fails // // worker_sandbox_versions.go builds the sandbox runtimes file for each // worker pod. @@ -39,12 +40,15 @@ import ( "strings" "sync" "time" + "unicode/utf8" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" + k8sruntime "k8s.io/apimachinery/pkg/runtime" coreinformers "k8s.io/client-go/informers/core/v1" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/events" "k8s.io/client-go/util/workqueue" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" @@ -92,6 +96,18 @@ const syncMaxRetries = 8 // cached syncs for other pods on the node. const syncWorkers = 2 +const ( + // eventReasonSandboxRuntimeSyncFailed is the reason of the Warning + // event emitted on a worker pod when its sandbox runtimes file cannot + // be synced (or is written without a restorable version). + eventReasonSandboxRuntimeSyncFailed = "SandboxRuntimeSyncFailed" + // eventActionSyncSandboxRuntimes is the action of events emitted while + // syncing a worker pod's sandbox runtimes file. + eventActionSyncSandboxRuntimes = "SyncSandboxRuntimes" + // maxEventNoteBytes is the events.k8s.io/v1 Note length limit (1 kB). + maxEventNoteBytes = 1024 +) + // workerSandboxSyncer keeps a sandbox runtimes file in each local worker's // directory under workersDir: the runtimes of the enabled versions of the // SandboxConfig the worker's pool uses. It creates a worker's directory once @@ -106,6 +122,9 @@ type workerSandboxSyncer struct { // so they are fetched before an actor asks for them and stay cached // while none runs them. prewarm runtimePrewarm + // recorder emits a Warning event on the worker pod when syncing its + // sandbox runtimes file fails. + recorder events.EventRecorder // pods holds this node's worker pods, indexed by podUIDIndex. pods cache.Indexer assets sandboxAssetFetcher @@ -139,7 +158,7 @@ type workerSandboxSyncer struct { microvmAssets map[string]bool } -func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, assets sandboxAssetFetcher, workersDir string, index *runtimeIndex, prewarm runtimePrewarm) *workerSandboxSyncer { +func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, assets sandboxAssetFetcher, workersDir string, index *runtimeIndex, prewarm runtimePrewarm, recorder events.EventRecorder) *workerSandboxSyncer { return &workerSandboxSyncer{ resolver: resolver, pods: pods, @@ -147,6 +166,7 @@ func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, scratchDir: ateletpath.SandboxAssetScratchDir, index: index, prewarm: prewarm, + recorder: recorder, // Backoff doubles from 1s for informer lag and unresolved pools, and // caps at 5m where uncapped retries for transient asset fetch or probe // failures stay until they recover. @@ -229,7 +249,7 @@ func (s *workerSandboxSyncer) syncDefaultRuntimeDirs(uid string, defaults []stri // or a list/watch fails (e.g. Forbidden while an RBAC rollout lags the // binary), existing files and index entries stay as they are and only // changes wait. -func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInformer, workerPools ateinformers.WorkerPoolInformer, configs ateinformers.SandboxConfigInformer, assets sandboxAssetFetcher, workersDir string, index *runtimeIndex, prewarm runtimePrewarm) (*workerSandboxSyncer, error) { +func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInformer, workerPools ateinformers.WorkerPoolInformer, configs ateinformers.SandboxConfigInformer, assets sandboxAssetFetcher, workersDir string, index *runtimeIndex, prewarm runtimePrewarm, recorder events.EventRecorder) (*workerSandboxSyncer, error) { if err := pods.Informer().AddIndexers(cache.Indexers{podUIDIndex: func(obj any) ([]string, error) { pod, ok := obj.(*corev1.Pod) if !ok { @@ -243,7 +263,7 @@ func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInforme workerPools: workerPools.Lister(), configs: configs.Lister(), } - s := newWorkerSandboxSyncer(resolver, pods.Informer().GetIndexer(), assets, workersDir, index, prewarm) + s := newWorkerSandboxSyncer(resolver, pods.Informer().GetIndexer(), assets, workersDir, index, prewarm, recorder) if err := watch(ctx, "worker pod", pods.Informer(), s.podHandler()); err != nil { return nil, err } @@ -444,6 +464,7 @@ func (s *workerSandboxSyncer) run(ctx context.Context, synced ...cache.InformerS // it is gone. func (s *workerSandboxSyncer) process(ctx context.Context, uid string) { defer s.queue.Done(uid) + var related k8sruntime.Object pod, err := s.podByUID(uid) switch { case err != nil: @@ -452,9 +473,9 @@ func (s *workerSandboxSyncer) process(ctx context.Context, uid string) { case pod.DeletionTimestamp != nil: // The Delete event removes the directory. default: - err = s.syncPod(ctx, pod) + related, err = s.syncPod(ctx, pod) } - s.finish(ctx, uid, err) + s.finish(ctx, pod, related, uid, err) } // podByUID returns the cached pod with the given UID, or nil if there is none. @@ -475,32 +496,69 @@ func (s *workerSandboxSyncer) podByUID(uid string) (*corev1.Pod, error) { } // finish forgets uid on success or errPermanent, caps errUnresolved retries at -// syncMaxRetries, and otherwise requeues with backoff without a retry cap. A -// sync that fails before it writes leaves the existing file in place. -func (s *workerSandboxSyncer) finish(ctx context.Context, uid string, err error) { +// syncMaxRetries, and otherwise requeues with backoff without a retry cap. It +// emits a Warning Event on pod for every failure except an unresolved pool +// still being retried. A sync that fails before it writes leaves the existing +// file in place. +func (s *workerSandboxSyncer) finish(ctx context.Context, pod *corev1.Pod, related k8sruntime.Object, uid string, err error) { switch { case err == nil: s.queue.Forget(uid) case errors.Is(err, errPermanent): slog.WarnContext(ctx, "Cannot sync worker sandbox runtimes; not retrying until the pool's SandboxConfig changes or atelet restarts", slog.String("pod_uid", uid), slog.Any("err", err)) + s.recordSyncFailure(pod, related, err) s.queue.Forget(uid) case ctx.Err() != nil: // Shutting down, not a sync failure. case errors.Is(err, errIncomplete): slog.WarnContext(ctx, "Worker sandbox runtimes file is missing a restorable version; will retry", slog.String("pod_uid", uid), slog.Int("retries", s.queue.NumRequeues(uid)), slog.Any("err", err)) + s.recordSyncFailure(pod, related, err) s.queue.AddRateLimited(uid) case s.queue.NumRequeues(uid) < syncMaxRetries || !errors.Is(err, errUnresolved): slog.WarnContext(ctx, "Could not sync worker sandbox runtimes; will retry", slog.String("pod_uid", uid), slog.Int("retries", s.queue.NumRequeues(uid)), slog.Any("err", err)) + if !errors.Is(err, errUnresolved) { + s.recordSyncFailure(pod, related, err) + } s.queue.AddRateLimited(uid) default: slog.WarnContext(ctx, "Could not sync worker sandbox runtimes; giving up until the pool's SandboxConfig changes or atelet restarts", slog.String("pod_uid", uid), slog.Int("retries", syncMaxRetries), slog.Any("err", err)) + s.recordSyncFailure(pod, related, err) s.queue.Forget(uid) } } +// recordSyncFailure emits a Warning event on pod when a sync fails. The +// broadcaster keys its series cache by (regarding, related, reason, action), +// including both objects' ResourceVersion, so retries against the same pod and +// SandboxConfig (or WorkerPool) revisions update the series count instead of +// creating a new Event on each attempt, while a pod status update, config edit, +// or config switch starts a new Event. A message that changes while both +// ResourceVersions stay the same still joins the open series and keeps the +// first attempt's Note. +func (s *workerSandboxSyncer) recordSyncFailure(pod *corev1.Pod, related k8sruntime.Object, err error) { + if pod == nil { + return + } + s.recorder.Eventf(pod, related, corev1.EventTypeWarning, eventReasonSandboxRuntimeSyncFailed, eventActionSyncSandboxRuntimes, "%s", truncateEventNote(err.Error())) +} + +// truncateEventNote caps msg at maxEventNoteBytes on a UTF-8 rune boundary, +// ending a truncated message with "...". +func truncateEventNote(msg string) string { + if len(msg) <= maxEventNoteBytes { + return msg + } + const ellipsis = "..." + cut := maxEventNoteBytes - len(ellipsis) + for cut > 0 && !utf8.RuneStart(msg[cut]) { + cut-- + } + return msg[:cut] + ellipsis +} + func (s *workerSandboxSyncer) removeDir(ctx context.Context, uid string) error { s.index.forget(uid) if err := os.RemoveAll(s.podDir(uid)); err != nil { diff --git a/cmd/atelet/worker_sandbox_syncer_test.go b/cmd/atelet/worker_sandbox_syncer_test.go index 0f0d1b108e..b82b07bdbc 100644 --- a/cmd/atelet/worker_sandbox_syncer_test.go +++ b/cmd/atelet/worker_sandbox_syncer_test.go @@ -25,6 +25,7 @@ import ( "testing" "testing/synctest" "time" + "unicode/utf8" "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" @@ -34,6 +35,8 @@ import ( kubeinformers "k8s.io/client-go/informers" kubefake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/events" + "k8s.io/client-go/tools/reference" "k8s.io/client-go/util/workqueue" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" @@ -171,7 +174,7 @@ func newTestSyncer(t *testing.T, objs ...any) *workerSandboxSyncer { s := newWorkerSandboxSyncer(&workerSandboxResolver{ workerPools: atelisters.NewWorkerPoolLister(pools), configs: atelisters.NewSandboxConfigLister(configs), - }, pods, newTestAssetFetcher(), t.TempDir(), newRuntimeIndex(), nil) + }, pods, newTestAssetFetcher(), t.TempDir(), newRuntimeIndex(), nil, events.NewFakeRecorder(64)) // The queue starts a goroutine; stop it with the test. t.Cleanup(s.queue.ShutDown) s.scratchDir = t.TempDir() // buildFile stages assets there @@ -389,7 +392,7 @@ func TestWorkerSandboxSyncerRegistersAheadOfJitter(t *testing.T) { s.sandboxConfigHandler().OnUpdate(multiVersionConfig(), cfg) synctest.Wait() - plan, err := s.resolver.plan(pod) + plan, _, err := s.resolver.plan(pod) if err != nil { t.Fatal(err) } @@ -414,6 +417,18 @@ func TestWorkerSandboxSyncerRegistersAheadOfJitter(t *testing.T) { }) } +// testEventRecorder wraps FakeRecorder to capture the related object passed to +// Eventf alongside the formatted event string. +type testEventRecorder struct { + *events.FakeRecorder + related k8sruntime.Object +} + +func (r *testEventRecorder) Eventf(regarding, related k8sruntime.Object, eventtype, reason, action, note string, args ...any) { + r.related = related + r.FakeRecorder.Eventf(regarding, related, eventtype, reason, action, note, args...) +} + func TestWorkerSandboxSyncerProcess(t *testing.T) { ctx := context.Background() deleting := workerPod("pod-a", "pool") @@ -439,6 +454,11 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { // indexed registers pod-a in the index before processing, as an // earlier sync would have; wantIndexed is whether it is there after. indexed, wantIndexed bool + // wantEvent is a substring of the Warning event emitted on pod-a, or "" + // when no event should be emitted. wantRelated is the secondary object + // attached to that event. + wantEvent string + wantRelated k8sruntime.Object }{{ name: "writes the file", wantIndexed: true, @@ -498,6 +518,13 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { name: "unresolved pool gives up after syncMaxRetries", objs: []any{workerPod("pod-a", "pool"), multiVersionConfig()}, priorRetries: syncMaxRetries, + wantEvent: "Warning SandboxRuntimeSyncFailed", + }, { + name: "unresolved pool config gives up after syncMaxRetries", + objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "")}, + priorRetries: syncMaxRetries, + wantEvent: "Warning SandboxRuntimeSyncFailed", + wantRelated: resolvedPool("pool", ""), }, { // A transient failure on the default version leaves the existing file // in place and requeues without the syncMaxRetries cap, since no @@ -515,6 +542,8 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { wantDir: true, wantUntouched: true, wantRetries: syncMaxRetries + 1, + wantEvent: "temporary download failure on default version", + wantRelated: multiVersionConfig(), }, { // A transient failure on a restorable version writes the default // runtime right away and requeues without the syncMaxRetries cap. @@ -533,13 +562,36 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { want: gvisorFile(wantRuntime("v2")), wantDir: true, wantRetries: syncMaxRetries + 1, + wantEvent: "temporary download failure on restorable version", + wantRelated: multiVersionConfig(), + }, { + name: "pod of no pool is not retried", + objs: []any{workerPod("pod-a", ""), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + wantEvent: "Warning SandboxRuntimeSyncFailed", }, { - name: "pod of no pool is not retried", - objs: []any{workerPod("pod-a", ""), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + name: "long sync error is truncated on a UTF-8 rune boundary to the Event note limit", + objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + existing: stale, + assets: &fakeAssetFetcher{ + fn: func(context.Context, *sandboxAssetsRecord) (map[string]string, error) { + // Prefix "SandboxConfig \"gvisor-release\", version \"v2\": " is 46 + // bytes, so 2-byte "é" runes start at even offsets and the + // 1021-byte cut point lands on a continuation byte. + return nil, errors.New(strings.Repeat("é", 1000)) + }, + }, + want: stale, + wantDir: true, + wantUntouched: true, + wantRetries: 1, + wantEvent: strings.Repeat("é", 487) + "...", + wantRelated: multiVersionConfig(), }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := newTestSyncer(t, tt.objs...) + recorder := &testEventRecorder{FakeRecorder: s.recorder.(*events.FakeRecorder)} + s.recorder = recorder if tt.assets != nil { s.assets = tt.assets } @@ -575,6 +627,37 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { s.process(ctx, uid) + var gotEvent string + select { + case gotEvent = <-recorder.Events: + default: + } + if tt.wantEvent == "" && gotEvent != "" { + t.Errorf("unexpected event %q", gotEvent) + } + if tt.wantEvent != "" && !strings.Contains(gotEvent, tt.wantEvent) { + t.Errorf("event = %q, want substring %q", gotEvent, tt.wantEvent) + } + if gotEvent != "" { + note := strings.TrimPrefix(gotEvent, "Warning SandboxRuntimeSyncFailed ") + if len(note) > maxEventNoteBytes || !utf8.ValidString(note) { + t.Errorf("event note len = %d (max %d), valid UTF-8 = %v", len(note), maxEventNoteBytes, utf8.ValidString(note)) + } + } + if diff := cmp.Diff(tt.wantRelated, recorder.related); diff != "" { + t.Errorf("event related mismatch (-want +got):\n%s", diff) + } + if recorder.related != nil { + if _, err := reference.GetReference(newEventScheme(), recorder.related); err != nil { + t.Errorf("reference.GetReference(%T) error = %v", recorder.related, err) + } + } + select { + case extra := <-recorder.Events: + t.Errorf("unexpected second event %q", extra) + default: + } + if _, err := os.Stat(dir); (err == nil) != tt.wantDir { t.Errorf("directory exists = %v (%v), want %v", err == nil, err, tt.wantDir) } @@ -851,7 +934,7 @@ func TestStartWorkerSandboxSyncer(t *testing.T) { ateFactory.Ate().V1().WorkerPools(), ateFactory.Ate().V1().SandboxConfigs(), newScriptAssetFetcher(t), - dir, newRuntimeIndex(), nil) + dir, newRuntimeIndex(), nil, &events.FakeRecorder{}) if err != nil { t.Fatalf("startWorkerSandboxSyncer() error = %v", err) } diff --git a/cmd/atelet/worker_sandbox_versions.go b/cmd/atelet/worker_sandbox_versions.go index 6449dcbba0..646286507d 100644 --- a/cmd/atelet/worker_sandbox_versions.go +++ b/cmd/atelet/worker_sandbox_versions.go @@ -43,6 +43,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" "go.ate.dev/substrate/internal/hardware" @@ -136,13 +137,20 @@ func (r *workerSandboxResolver) config(pool *atev1.WorkerPool) (*atev1.SandboxCo return cfg, nil } -// plan resolves pod's SandboxConfig and the versions of it this node can run. -func (r *workerSandboxResolver) plan(pod *corev1.Pod) (*sandboxPlan, error) { +// plan resolves pod's SandboxConfig and the versions of it this node can run, +// along with the object the sync is acting on (the resolved SandboxConfig, or +// the WorkerPool when its config has not resolved) for failure Events. +func (r *workerSandboxResolver) plan(pod *corev1.Pod) (*sandboxPlan, k8sruntime.Object, error) { pool, err := r.pool(pod) if err != nil { - return nil, err + return nil, nil, err } - return r.planPool(pool) + cfg, err := r.config(pool) + if err != nil { + return nil, pool, err + } + plan, err := planConfig(cfg) + return plan, cfg, err } // planPool is plan for a pool already in hand: the config its workers run @@ -152,6 +160,12 @@ func (r *workerSandboxResolver) planPool(pool *atev1.WorkerPool) (*sandboxPlan, if err != nil { return nil, err } + return planConfig(cfg) +} + +// planConfig returns the plan for cfg: its class and the versions of it this +// node can run. +func planConfig(cfg *atev1.SandboxConfig) (*sandboxPlan, error) { vers, err := planVersions(cfg) if err != nil { return nil, err @@ -486,30 +500,32 @@ func bundleHash(rec *sandboxAssetsRecord) string { // // A file missing restorable runtimes for a transient reason is written all // the same, and the errIncomplete from build is returned so finish retries. -func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) error { - plan, err := s.resolver.plan(pod) +// The returned k8sruntime.Object is the resolved SandboxConfig (or WorkerPool +// when its config has not resolved), attached as `related` on failure Events. +func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) (k8sruntime.Object, error) { + plan, related, err := s.resolver.plan(pod) if err != nil { - return err + return related, err } hashed := hashVersions(plan) // Resolvable, and on disk, before the file can let the worker register // any of them. if err := s.index.registerRuntimes(plan.config, hashed); err != nil { - return err + return related, err } want, buildErr := s.buildFile(ctx, plan, hashed) if buildErr != nil && !errors.Is(buildErr, errIncomplete) { - return buildErr + return related, buildErr } if err := s.write(ctx, string(pod.UID), plan.config, want); err != nil { - return err + return related, err } // What the worker advertises is what its file now says: a sync that // failed before the write left the previous entry, and one that wrote // a partial file names only the runtimes in it. One class per pool // today; a pool that runs several will yield one plan per class here. if err := s.index.register(string(pod.UID), advertisedClass{class: plan.sandboxClass, config: plan.config, hashed: writtenVersions(hashed, want)}); err != nil { - return err + return related, err } // Only now, with the file naming its defaults: a sync that failed above // left the file, and so the directories, as they were. The build staged @@ -524,13 +540,13 @@ func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) erro } } if err := s.syncDefaultRuntimeDirs(string(pod.UID), defaults); err != nil { - return err + return related, err } for _, hash := range defaults { s.prewarm.enqueue(hash, s.defaultRuntimeDir(string(pod.UID), hash)) } } - return buildErr + return related, buildErr } // writtenVersions returns the versions of hashed whose runtime file names, diff --git a/cmd/atelet/worker_sandbox_versions_test.go b/cmd/atelet/worker_sandbox_versions_test.go index 827ad39c17..100a9ae3d7 100644 --- a/cmd/atelet/worker_sandbox_versions_test.go +++ b/cmd/atelet/worker_sandbox_versions_test.go @@ -266,7 +266,7 @@ func wantVersions() *sandboxruntimes.File { // planAndBuild runs the plan and build steps of syncPod for pod. func planAndBuild(s *workerSandboxSyncer, pod *corev1.Pod) (*sandboxruntimes.File, error) { - plan, err := s.resolver.plan(pod) + plan, _, err := s.resolver.plan(pod) if err != nil { return nil, err } @@ -541,7 +541,7 @@ func TestSyncPodMicroVMAssetCacheAndEviction(t *testing.T) { recPrewarm := &recordingPrewarm{} s.prewarm = recPrewarm - if err := s.syncPod(ctx, workerPod("pod-a", "microvm-pool")); err != nil { + if _, err := s.syncPod(ctx, workerPod("pod-a", "microvm-pool")); err != nil { t.Fatalf("syncPod(pod-a): %v", err) } if left, err := os.ReadDir(s.scratchDir); err != nil || len(left) != 0 { @@ -575,7 +575,7 @@ func TestSyncPodMicroVMAssetCacheAndEviction(t *testing.T) { storage.mu.Unlock() for _, uid := range []string{"pod-b", "pod-a"} { - if err := s.syncPod(ctx, workerPod(uid, "microvm-pool")); err != nil { + if _, err := s.syncPod(ctx, workerPod(uid, "microvm-pool")); err != nil { t.Fatalf("syncPod(%s) after eviction with bucket offline: %v", uid, err) } got, err := sandboxruntimes.Read(s.runtimesFile(uid)) @@ -593,7 +593,7 @@ func TestSyncPodMicroVMAssetCacheAndEviction(t *testing.T) { // Promoting the evicted v1 to default while the bucket is offline must fail // the sync and keep pod-a's previous file and default-runtime pin in place. cfg.Spec.DefaultVersion = "v1" - if err := s.syncPod(ctx, workerPod("pod-a", "microvm-pool")); err == nil { + if _, err := s.syncPod(ctx, workerPod("pod-a", "microvm-pool")); err == nil { t.Fatal("syncPod(pod-a) after promoting evicted v1 to default with bucket offline = nil, want error") } got, err := sandboxruntimes.Read(s.runtimesFile("pod-a")) @@ -847,7 +847,7 @@ func TestSyncPodAdvertisesWhatItWrote(t *testing.T) { t.Run("a failed sync keeps the entry of the file on disk", func(t *testing.T) { s := newTestSyncer(t, resolvedPool("pool", "gvisor-release"), multiVersionConfig()) - if err := s.syncPod(ctx, pod); err != nil { + if _, err := s.syncPod(ctx, pod); err != nil { t.Fatalf("syncPod: %v", err) } before := advertised(t, s) @@ -860,7 +860,7 @@ func TestSyncPodAdvertisesWhatItWrote(t *testing.T) { s.cpuFeatures = map[string]string{} s.mu.Unlock() s.runCPUFeatures = func(context.Context, string) ([]string, error) { return []string{"not_a_known_feature"}, nil } - if err := s.syncPod(ctx, pod); !errors.Is(err, errPermanent) { + if _, err := s.syncPod(ctx, pod); !errors.Is(err, errPermanent) { t.Fatalf("syncPod with the default's probe failing = %v, want errPermanent", err) } if after := advertised(t, s); after.defaultHash != before.defaultHash || !slices.Equal(after.hashes, before.hashes) { @@ -878,7 +878,7 @@ func TestSyncPodAdvertisesWhatItWrote(t *testing.T) { return map[string]string{gvisorAssetName: "sha-v2"}, nil }, } - if err := s.syncPod(ctx, pod); !errors.Is(err, errIncomplete) { + if _, err := s.syncPod(ctx, pod); !errors.Is(err, errIncomplete) { t.Fatalf("syncPod with a restorable version failing = %v, want errIncomplete", err) } if cr := advertised(t, s); cr.defaultHash != hashOf("v2") || !slices.Equal(cr.hashes, []string{hashOf("v2")}) { diff --git a/cmd/atelet/workersupport.go b/cmd/atelet/workersupport.go index 44fd8dab52..60ca5f6b8d 100644 --- a/cmd/atelet/workersupport.go +++ b/cmd/atelet/workersupport.go @@ -221,7 +221,19 @@ func toAteAPISandboxRuntime(in *ateletpb.WorkerSandboxRuntime) *ateapi.WorkerSan if in == nil { return nil } - return &ateapi.WorkerSandboxRuntime{Version: toAteAPICompat(in.GetVersion())} + return &ateapi.WorkerSandboxRuntime{ + Version: toAteAPICompat(in.GetVersion()), + ActorOverhead: toAteAPIOverhead(in.GetActorOverhead()), + } +} + +// toAteAPIOverhead converts atelet's RuntimeOverhead to the control plane's, +// which it mirrors field for field. +func toAteAPIOverhead(in *ateletpb.RuntimeOverhead) *ateapi.RuntimeOverhead { + if in == nil { + return nil + } + return &ateapi.RuntimeOverhead{MemoryBytes: in.GetMemoryBytes(), MemoryBytesPerGib: in.GetMemoryBytesPerGib()} } func toAteAPISandboxRuntimes(in []*ateletpb.WorkerSandboxRuntime) []*ateapi.WorkerSandboxRuntime { diff --git a/cmd/atelet/workersupport_test.go b/cmd/atelet/workersupport_test.go index 73d2c42972..8e596fddc1 100644 --- a/cmd/atelet/workersupport_test.go +++ b/cmd/atelet/workersupport_test.go @@ -325,6 +325,21 @@ func TestSetWorkerSandboxRuntimes(t *testing.T) { name: "records what the worker says", req: &ateletpb.SetWorkerSandboxRuntimesRequest{Epoch: 3, SandboxClasses: reqClasses()}, wantForwarded: forwarded(), + }, { + name: "forwards the runtimes' actor overhead", + req: func() *ateletpb.SetWorkerSandboxRuntimesRequest { + classes := reqClasses(reqCompat("d547")) + classes[0].DefaultRuntime.ActorOverhead = &ateletpb.RuntimeOverhead{MemoryBytes: 4 << 20, MemoryBytesPerGib: 8 << 20} + classes[0].RestorableRuntimes[0].ActorOverhead = &ateletpb.RuntimeOverhead{MemoryBytes: 16 << 20} + return &ateletpb.SetWorkerSandboxRuntimesRequest{Epoch: 3, SandboxClasses: classes} + }(), + wantForwarded: func() []*ateapi.SetWorkerSandboxRuntimesRequest { + want := forwarded(wantCompat("d547")) + c := want[0].GetSandboxClasses()[0] + c.DefaultRuntime.ActorOverhead = &ateapi.RuntimeOverhead{MemoryBytes: 4 << 20, MemoryBytesPerGib: 8 << 20} + c.RestorableRuntimes[0].ActorOverhead = &ateapi.RuntimeOverhead{MemoryBytes: 16 << 20} + return want + }(), }, { name: "forwards restorable runtimes", req: &ateletpb.SetWorkerSandboxRuntimesRequest{Epoch: 3, SandboxClasses: reqClasses(reqCompat("d547"))}, diff --git a/cmd/atenet/README.md b/cmd/atenet/README.md index 3abe9e1a0e..afbc58a6ee 100644 --- a/cmd/atenet/README.md +++ b/cmd/atenet/README.md @@ -39,7 +39,7 @@ likely be split in the future for better scalability.) requests finish instead of resetting. The whole sequence must fit within `terminationGracePeriodSeconds` (see the manifest comments). An upgrade rolls the router as an ordinary Deployment update (see - [docs/upgrade.md](../../docs/upgrade.md)). The drain lets each old pod's + [docs/guides/upgrade.md](../../docs/guides/upgrade.md)). The drain lets each old pod's requests finish, except one still open about 15 seconds after the drain starts, which is cut. diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index 2d7076bfa7..3150638ef0 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -15,7 +15,7 @@ Router has several responsibilities: * Parks requests whose actor cannot be served immediately due to transient worker-pool saturation, retrying the resume until the actor is routable or a bounded wait elapses, instead of failing fast. See - [docs/request-parking.md](../../../../docs/request-parking.md). + [docs/concepts/request-parking.md](../../../../docs/concepts/request-parking.md). * Drains gracefully on SIGTERM: flips `/readyz` so the Service stops sending new connections, waits out endpoint propagation (`--drain-delay`), drains the dataplane's established connections (Envoy only — driven over its admin API; @@ -105,7 +105,7 @@ rule, `egress_passthrough` under a `tls_passthrough` rule, and `denied`, which matches no chain and closes the connection, when neither covers the SNI. The passthrough chain resolves the SNI and relays the bytes to it on the dialed port; the address the actor dialed plays no part (see -`docs/network-egress.md`). +`docs/guides/network-egress.md`). Identity on the request legs is `dev.ate.actor.identity`, the actor's SPIFFE ID that the outer chain set from the verified peer certificate and shares with @@ -125,7 +125,7 @@ deleted policy becomes a deny. Every policy denial answers a fixed Credential injection (`replace_headers`) runs on both request legs: a header the request carries is replaced with the credential from the provider, and a request without it is forwarded unchanged. See -`docs/egress-credential-injection.md`. +`docs/guides/egress-credential-injection.md`. ## adding a dataplane attribute @@ -216,7 +216,8 @@ Right now the only "authorization" policy is to check the client's SPIFFE ID against a static allowlist. Any ID on the allowlist may use the ingress router to contact any actor. Richer authorization policies may be added post-GA. Use the `--ingress-allowed-spiffe-ids` flag to set the allowlist (comma-separated -list). +list). With an empty allowlist, the Envoy router serves no listener, so no +traffic reaches actors through it. ### Common setups @@ -257,15 +258,22 @@ volumes: Add the pod's SPIFFE ID to `--ingress-allowed-spiffe-ids`. -**Client outside the cluster, pod identity CA.** A demo on your machine can -follow the e2e tests in `internal/e2e/ingressauth.go`. It creates a -PodCertificateRequest for the podidentity signer that names a pod, node and -ServiceAccount that don't exist. kube-apiserver checks those only for requests -from a kubelet, and the signer issues the SPIFFE ID of the named namespace and -ServiceAccount. The client keeps the private key, reads the chain from the -request's status, and reads the servicedns ClusterTrustBundle to verify the -router. Creating PodCertificateRequests needs RBAC for that resource in the -namespace. +**Client outside the cluster, pod identity CA.** `kubectl ate proxy actor` +forwards local TCP ports to an actor through the router's CONNECT listener. It +presents a certificate for +`spiffe://cluster.local/ns/ate-system/sa/atenet-client`, so that ID must be in +`--ingress-allowed-spiffe-ids`. It requests the certificate the way +`internal/podidentity` does: a PodCertificateRequest for the podidentity +signer that names a pod, node and ServiceAccount that don't exist. +kube-apiserver checks those only for requests from a kubelet, and the signer +issues the SPIFFE ID of the named namespace and ServiceAccount. The client +keeps the private key, reads the chain from the request's status, and reads +the servicedns ClusterTrustBundle to verify the router. + +Creating PodCertificateRequests needs RBAC for that resource in `ate-system`. +That RBAC can request a certificate for any SPIFFE ID in `ate-system`, +including those of atelet, ate-controller and ate-api-server, so grant it only +to users you would trust with those components. ## status page diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index 17620d57fe..90f02d1a0d 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -72,7 +72,7 @@ func bindRouterFlags(fs *pflag.FlagSet, cfg *routerConfig) { // Flags for client authentication and authorization in ingress mode. fs.StringVar(&cfg.IngressAuth.ClientCAFile, "ingress-client-ca-file", "", "PEM bundle of the CAs allowed to sign ingress client certificates. Envoy reads the file, so it must be mounted at this path in the Envoy container as well. Required for the Envoy ingress dataplane") - fs.StringSliceVar(&cfg.IngressAuth.AllowedSPIFFEIDs, "ingress-allowed-spiffe-ids", nil, "SPIFFE IDs (exact match) allowed to connect to actors through the ingress dataplane. Comma-separated or repeated. Required for the Envoy ingress dataplane") + fs.StringSliceVar(&cfg.IngressAuth.AllowedSPIFFEIDs, "ingress-allowed-spiffe-ids", nil, "SPIFFE IDs (exact match) allowed to connect to actors through the ingress dataplane. Comma-separated or repeated. Without any, the Envoy ingress dataplane serves no traffic") fs.StringVar(&cfg.UpstreamCredentialBundlePath, "upstream-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle (cert+key) the router presents as the client cert when dialing the actor's atunnel ingress server over mTLS. Empty disables upstream mTLS (legacy plaintext pod-IP:80) and needs --upstream-connect=false.") fs.StringVar(&cfg.UpstreamTrustBundlePath, "upstream-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to validate the actor's atunnel ingress server certificate.") @@ -83,14 +83,14 @@ func bindRouterFlags(fs *pflag.FlagSet, cfg *routerConfig) { fs.DurationVar(&cfg.EgressPolicyCacheTTL, "egress-policy-cache-ttl", egress.DefaultPolicyCacheTTL, "How long the egress gateway keeps acting on an actor's EgressPolicy before fetching it from ateapi again, which bounds the lag between a policy change and its effect on new requests. 0 disables the cache (concurrent callouts for one actor still share a fetch)") // Egress credential injection (MITM leg). Only the egress gateway sets these, - // and only when injection is enabled: with an empty + // and only when injection is enabled: with no // --credential-provider-address, a request matching an EgressPolicy rule // that requires injection is denied. - fs.StringVar(&cfg.CredentialProvider.Name, "credential-provider-name", "", "Credential provider this egress gateway serves, as the host of its ate-secret:// credential URIs (e.g. k8s.io); a policy credential URI naming any other provider is refused. Empty disables the check (dev only)") - fs.StringVar(&cfg.CredentialProvider.Address, "credential-provider-address", "", "gRPC dial target of the credential provider the MITM-leg injector resolves secrets through. Empty (the default) disables egress credential injection, and requests matching a rule that requires it are denied") + fs.StringArrayVar(&cfg.CredentialProvider.Names, "credential-provider-name", nil, "Credential provider this egress gateway serves, as the host of its ate-secret:// credential URIs (e.g. k8s.io). Repeat once per --credential-provider-address, in the same order; a policy credential URI naming no configured provider is refused. A single empty name disables the check (dev only)") + fs.StringArrayVar(&cfg.CredentialProvider.Addresses, "credential-provider-address", nil, "gRPC dial target of a credential provider the MITM-leg injector resolves secrets through. Repeat once per provider. None (the default) disables egress credential injection, and requests matching a rule that requires it are denied") fs.StringVar(&cfg.CredentialProvider.CAFile, "credential-provider-ca-file", "", "CA the credential provider's serving certificate must chain to; required unless --credential-provider-insecure is set") fs.StringVar(&cfg.CredentialProvider.ClientCert, "credential-provider-client-cert", "", "Credential bundle presented to the credential provider as the client certificate; required unless --credential-provider-insecure is set") - fs.StringVar(&cfg.CredentialProvider.ServerName, "credential-provider-server-name", "", "SAN/SNI expected on the credential provider's serving certificate") + fs.StringArrayVar(&cfg.CredentialProvider.ServerNames, "credential-provider-server-name", nil, "SAN/SNI expected on a credential provider's serving certificate. Repeat once per --credential-provider-address, in the same order, or omit") fs.BoolVar(&cfg.CredentialProvider.Insecure, "credential-provider-insecure", false, "Dial the credential provider WITHOUT TLS. Development only: secrets cross the network in the clear. Without this, a missing --credential-provider-ca-file or --credential-provider-client-cert fails startup instead of silently downgrading") // Envoy learns the collector over xDS rather than from its own environment, diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 5345caef7d..61527158c6 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -16,6 +16,7 @@ package router import ( "fmt" + "slices" "time" "github.com/spiffe/go-spiffe/v2/spiffeid" @@ -73,7 +74,7 @@ type ingressAuthConfig struct { // container at the same path. ClientCAFile string // AllowedSPIFFEIDs is the exact set of SPIFFE IDs allowed to connect to - // actors. + // actors. When it is empty, the dataplane serves no traffic. // // Entries are complete SPIFFE IDs, like // `spiffe://mytrustdomain.com/foo/bar/baz`. @@ -87,29 +88,52 @@ type authConfig struct { AteapiServerName string } -// credentialProviderConfig holds the egress gateway's connection to the -// credential provider the MITM-leg injection resolves secrets through. A -// non-empty Address enables injection; the rest configures the mTLS the gateway -// dials the provider with. See cmd/atenet/internal/router/egress. +// credentialProviderConfig holds the egress gateway's connections to the +// credential providers the MITM-leg injection resolves secrets through. Names, +// Addresses and ServerNames pair up by position; the rest configures the mTLS +// the gateway dials every provider with. See cmd/atenet/internal/router/egress. type credentialProviderConfig struct { - // Name is the provider this gateway serves, as a ate-secret:// - // prefix; a policy credential URI naming any other provider is refused. - // Empty disables the check (dev only). - Name string - // Address is the provider's gRPC dial target. Empty disables injection. - Address string + // Names are the providers this gateway serves, as ate-secret:// + // prefixes; a policy credential URI naming any other provider is refused. + // A lone empty name disables the check (dev only). + Names []string + // Addresses are the providers' gRPC dial targets. None disables injection. + Addresses []string // CAFile is the CA the provider's serving certificate must chain to. // Required unless Insecure is set. CAFile string // ClientCert is the credential bundle presented to the provider as the // client certificate. Required unless Insecure is set. ClientCert string - // ServerName is the SAN/SNI expected on the provider's serving certificate. - ServerName string + // ServerNames are the SAN/SNI expected on each provider's serving + // certificate. Empty leaves them all unset. + ServerNames []string // Insecure dials the provider without TLS; explicit development-only opt-in. Insecure bool } +// validate checks that the repeated provider flags pair up. +func (p credentialProviderConfig) validate() error { + if len(p.Names) != len(p.Addresses) { + return fmt.Errorf("got %d --credential-provider-name for %d --credential-provider-address, want one per address", len(p.Names), len(p.Addresses)) + } + if len(p.ServerNames) != 0 && len(p.ServerNames) != len(p.Addresses) { + return fmt.Errorf("got %d --credential-provider-server-name for %d --credential-provider-address, want one per address or none", len(p.ServerNames), len(p.Addresses)) + } + for i, name := range p.Names { + if p.Addresses[i] == "" { + return fmt.Errorf("--credential-provider-address must not be empty") + } + if name == "" && len(p.Names) > 1 { + return fmt.Errorf("an empty --credential-provider-name serves every credential URI, so it must be the only provider") + } + if slices.Contains(p.Names[:i], name) { + return fmt.Errorf("--credential-provider-name %q repeats", name) + } + } + return nil +} + // routerConfig holds deployment setup and endpoint options for the router node instance. type routerConfig struct { // Mode is the one traffic direction this instance serves. Required. @@ -156,10 +180,9 @@ type routerConfig struct { EgressPolicyCacheTTL time.Duration // CredentialProvider configures egress credential injection on the MITM leg. - // Only the egress gateway sets it, and only when injection is enabled: an - // empty CredentialProvider.Address leaves the injector disabled, so a rule - // that requires an injection is skipped and the request passes through - // without the credential. See egress.Handler. + // Only the egress gateway sets it, and only when injection is enabled: a + // CredentialProvider with no Addresses leaves the injector disabled, so a + // request that needs an injection is denied. See egress.Handler. CredentialProvider credentialProviderConfig LogLevel string @@ -281,6 +304,9 @@ func (c routerConfig) validate() error { if err := c.validateIngressAuth(); err != nil { return err } + if err := c.CredentialProvider.validate(); err != nil { + return err + } if c.ExtProcMaxRequests < 0 { return fmt.Errorf("--extproc-max-requests must not be negative, got %d (0 derives it from --parked-request-max)", c.ExtProcMaxRequests) @@ -326,9 +352,6 @@ func (c routerConfig) validateIngressAuth() error { if c.IngressAuth.ClientCAFile == "" { return fmt.Errorf("--ingress-client-ca-file is required") } - if len(c.IngressAuth.AllowedSPIFFEIDs) == 0 { - return fmt.Errorf("at least one --ingress-allowed-spiffe-ids entry is required") - } for _, id := range c.IngressAuth.AllowedSPIFFEIDs { if _, err := spiffeid.FromString(id); err != nil { return fmt.Errorf("--ingress-allowed-spiffe-ids: %q is not a valid SPIFFE ID: %w", id, err) diff --git a/cmd/atenet/internal/router/config_test.go b/cmd/atenet/internal/router/config_test.go index d047cee62a..746e563878 100644 --- a/cmd/atenet/internal/router/config_test.go +++ b/cmd/atenet/internal/router/config_test.go @@ -47,6 +47,43 @@ func TestRouterConfigValidate(t *testing.T) { cfg: routerConfig{Mode: ModeIngress, AtenetRouter: "blah"}, wantErr: "--atenet-dataplane must be", }, + { + name: "credential providers paired by position are valid", + cfg: routerConfig{Mode: ModeEgress, CredentialProvider: credentialProviderConfig{ + Names: []string{"k8s.io", "sts.example.com"}, Addresses: []string{"a:1", "b:1"}, ServerNames: []string{"a", "b"}, + }}, + }, + { + name: "empty credential provider address rejected", + cfg: routerConfig{Mode: ModeEgress, CredentialProvider: credentialProviderConfig{Names: []string{"k8s.io"}, Addresses: []string{""}}}, + wantErr: "--credential-provider-address must not be empty", + }, + { + name: "credential provider name without an address rejected", + cfg: routerConfig{Mode: ModeEgress, CredentialProvider: credentialProviderConfig{Names: []string{"k8s.io"}}}, + wantErr: "want one per address", + }, + { + name: "credential provider server names not paired rejected", + cfg: routerConfig{Mode: ModeEgress, CredentialProvider: credentialProviderConfig{ + Names: []string{"k8s.io", "sts.example.com"}, Addresses: []string{"a:1", "b:1"}, ServerNames: []string{"a"}, + }}, + wantErr: "want one per address or none", + }, + { + name: "repeated credential provider name rejected", + cfg: routerConfig{Mode: ModeEgress, CredentialProvider: credentialProviderConfig{ + Names: []string{"k8s.io", "k8s.io"}, Addresses: []string{"a:1", "b:1"}, + }}, + wantErr: `"k8s.io" repeats`, + }, + { + name: "empty credential provider name next to another rejected", + cfg: routerConfig{Mode: ModeEgress, CredentialProvider: credentialProviderConfig{ + Names: []string{"", "k8s.io"}, Addresses: []string{"a:1", "b:1"}, + }}, + wantErr: "must be the only provider", + }, { name: "negative extproc-max-requests rejected", cfg: routerConfig{Mode: ModeIngress, ExtProcMaxRequests: -1, ParkedRequest: ingress.ParkedRequestConfig{Max: 0}}, @@ -219,9 +256,8 @@ func TestRouterConfigValidateIngressAuth(t *testing.T) { wantErr: "--ingress-client-ca-file is required", }, { - name: "empty allowlist rejected", - mutate: func(c *routerConfig) { c.IngressAuth.AllowedSPIFFEIDs = nil }, - wantErr: "at least one --ingress-allowed-spiffe-ids entry is required", + name: "empty allowlist accepted", + mutate: func(c *routerConfig) { c.IngressAuth.AllowedSPIFFEIDs = nil }, }, { name: "non-SPIFFE ID rejected", diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index 8e455ff8f1..3ce608ce32 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -109,9 +109,13 @@ func (s *RouterServer) newEnvoyXdsServer(ctx context.Context, parkCfg ingress.Pa return nil, fmt.Errorf("loading --ingress-client-ca-file: %w", err) } xdsSrv.SetDownstreamClientAuth(s.cfg.IngressAuth.ClientCAFile, s.cfg.IngressAuth.AllowedSPIFFEIDs) - slog.InfoContext(ctx, "TLS ingress listeners require client certificates", - slog.String("client_ca_file", s.cfg.IngressAuth.ClientCAFile), - slog.Any("allowed_spiffe_ids", s.cfg.IngressAuth.AllowedSPIFFEIDs)) + if len(s.cfg.IngressAuth.AllowedSPIFFEIDs) == 0 { + slog.InfoContext(ctx, "No --ingress-allowed-spiffe-ids, so the router serves no traffic") + } else { + slog.InfoContext(ctx, "TLS ingress listeners require client certificates", + slog.String("client_ca_file", s.cfg.IngressAuth.ClientCAFile), + slog.Any("allowed_spiffe_ids", s.cfg.IngressAuth.AllowedSPIFFEIDs)) + } xdsSrv.SetUpstreamTls(s.cfg.UpstreamCredentialBundlePath, s.cfg.UpstreamTrustBundlePath) xdsSrv.SetUpstreamConnect(s.cfg.UpstreamConnect) diff --git a/cmd/atenet/internal/router/egress/credentials.go b/cmd/atenet/internal/router/egress/credentials.go index 4814a155ad..85bca5063b 100644 --- a/cmd/atenet/internal/router/egress/credentials.go +++ b/cmd/atenet/internal/router/egress/credentials.go @@ -18,6 +18,8 @@ import ( "context" "fmt" "log/slog" + "maps" + "slices" "strings" "time" @@ -139,34 +141,36 @@ func (h *Handler) actorJWT(ctx context.Context, ref resources.ActorRef, dest egr // providerCredential fetches the secret uri names from the credential // provider. Every error it returns is already a client-facing denial. func (h *Handler) providerCredential(ctx context.Context, ref resources.ActorRef, dest egresspolicy.Destination, uri string) ([]byte, error) { - if h.provider == nil { + if len(h.providers) == 0 { slog.ErrorContext(ctx, "egress denied: policy requires credential injection but no credential provider is configured", slog.Any("actor", ref), slog.String("host", dest.Hostname)) return nil, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, deniedBody) } - // Confirm the credential URI names the provider this gateway serves - // before dialing: the configured connection fronts one provider, so a URI - // naming another cannot be resolved here and must fail closed rather than - // be sent to the wrong provider. - if h.providerName != "" { + // When h.providers has exactly one entry and its key is "", that provider + // serves every URI. Otherwise look up the provider named by the URI's host, + // and fail closed if none matches. + var provider credprovider.CredentialProviderClient + if p, ok := h.providers[""]; ok && len(h.providers) == 1 { + provider = p + } else { name, err := providerNameFromURI(uri) if err != nil { slog.ErrorContext(ctx, "egress denied: policy names an unparseable credential URI", slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", uri), slog.Any("err", err)) return nil, extproc.WrapReqError(envoy_type.StatusCode_InternalServerError, err, deniedBody) } - if name != h.providerName { + if provider = h.providers[name]; provider == nil { slog.ErrorContext(ctx, "egress denied: credential URI names a provider this gateway does not serve", slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", uri), - slog.String("provider", name), slog.String("serves", h.providerName)) + slog.String("provider", name), slog.Any("serves", slices.Sorted(maps.Keys(h.providers)))) return nil, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, deniedBody) } } // Atunnel connected to us with a worker-for-actor SPIFFE ID; translate it // to a pure actor SPIFFE ID for plugins to make decisions on. - resp, err := h.provider.FetchSecret(ctx, &credprovider.FetchSecretRequest{ + resp, err := provider.FetchSecret(ctx, &credprovider.FetchSecretRequest{ Uri: uri, ActorSpiffeId: resources.ActorSPIFFEID(ref).String(), }) diff --git a/cmd/atenet/internal/router/egress/credentials_test.go b/cmd/atenet/internal/router/egress/credentials_test.go index b69fb4af84..0d5bc731fa 100644 --- a/cmd/atenet/internal/router/egress/credentials_test.go +++ b/cmd/atenet/internal/router/egress/credentials_test.go @@ -18,6 +18,7 @@ import ( "cmp" "context" "errors" + "maps" "testing" "testing/synctest" "time" @@ -57,7 +58,11 @@ func bearerTokenResponse(token string) *credprovider.FetchSecretResponse { } func injectionHandlerFor(policy *ateapi.EgressPolicy, provider credprovider.CredentialProviderClient, providerName string) *Handler { - return New(&egressMockClient{actor: runningActor(), policy: policy}, nil, 0, provider, providerName) + var providers map[string]credprovider.CredentialProviderClient + if provider != nil { + providers = map[string]credprovider.CredentialProviderClient{providerName: provider} + } + return New(&egressMockClient{actor: runningActor(), policy: policy}, nil, 0, providers) } // withAuthorization adds the placeholder authorization header the sample @@ -113,6 +118,69 @@ func TestInjectionOnRequestLegs(t *testing.T) { } } +func TestInjectionPicksProviderByName(t *testing.T) { + policy := &ateapi.EgressPolicy{Rules: []*ateapi.EgressRule{{Https: &ateapi.HttpsRule{ + Hostnames: []string{"api.example.com"}, + Effects: &ateapi.HttpRuleEffects{ReplaceHeaders: []*ateapi.CredentialHeader{ + {Header: "authorization", Prefix: "Bearer ", CredentialUri: "ate-secret://k8s/default/token"}, + {Header: "x-api-key", CredentialUri: "ate-secret://vault/team/key"}, + }}, + }}}} + k8s := &fakeProvider{resp: bearerTokenResponse("from-k8s")} + vault := &fakeProvider{resp: bearerTokenResponse("from-vault")} + h := New(&egressMockClient{actor: runningActor(), policy: policy}, nil, 0, + map[string]credprovider.CredentialProviderClient{"k8s": k8s, "vault": vault}) + + md := withAuthorization(innerMetadata(extproc.EgressTLSMITMFilterChainName, "GET", "api.example.com", nil)) + md.Headers["x-api-key"] = "placeholder" + res, err := h.HandleRequestHeaders(context.Background(), md) + if err != nil { + t.Fatalf("HandleRequestHeaders: %v", err) + } + got := map[string]string{} + for _, hv := range res.Response.GetResponse().GetHeaderMutation().GetSetHeaders() { + got[hv.GetHeader().GetKey()] = string(hv.GetHeader().GetRawValue()) + } + want := map[string]string{"authorization": "Bearer from-k8s", "x-api-key": "from-vault"} + if !maps.Equal(got, want) { + t.Errorf("injected headers = %v, want %v", got, want) + } + if uri := k8s.got.GetUri(); uri != "ate-secret://k8s/default/token" { + t.Errorf("k8s provider got URI %q", uri) + } + if uri := vault.got.GetUri(); uri != "ate-secret://vault/team/key" { + t.Errorf("vault provider got URI %q", uri) + } +} + +func TestInjectionCatchAllProvider(t *testing.T) { + policy := credentialInjectionPolicySample("api.example.com") + md := func() *extproc.RequestMetadata { + return withAuthorization(innerMetadata(extproc.EgressTLSMITMFilterChainName, "GET", "api.example.com", nil)) + } + + catchAll := &fakeProvider{resp: bearerTokenResponse("s3cr3t")} + h := New(&egressMockClient{actor: runningActor(), policy: policy}, nil, 0, + map[string]credprovider.CredentialProviderClient{"": catchAll}) + if _, err := h.HandleRequestHeaders(context.Background(), md()); err != nil { + t.Fatalf("HandleRequestHeaders with a lone \"\" provider: %v", err) + } + if catchAll.got == nil { + t.Error("a lone \"\" provider was not asked for the credential") + } + + catchAll = &fakeProvider{resp: bearerTokenResponse("s3cr3t")} + k8s := &fakeProvider{resp: bearerTokenResponse("s3cr3t")} + h = New(&egressMockClient{actor: runningActor(), policy: policy}, nil, 0, + map[string]credprovider.CredentialProviderClient{"": catchAll, injectionProviderName: k8s}) + if _, err := h.HandleRequestHeaders(context.Background(), md()); err != nil { + t.Fatalf("HandleRequestHeaders with a \"\" provider next to another: %v", err) + } + if catchAll.got != nil || k8s.got == nil { + t.Errorf("next to another provider, the \"\" entry was asked (%v) and %s was asked (%v); want only %s", catchAll.got != nil, injectionProviderName, k8s.got != nil, injectionProviderName) + } +} + // A request that does not carry the header a rule replaces goes out unchanged: // nothing is fetched, added, or denied, even where injecting would fail. func TestInjectionSkippedWithoutHeader(t *testing.T) { @@ -312,7 +380,7 @@ func TestActorJWTInjection(t *testing.T) { actorJWTHeader("authorization"), &ateapi.CredentialHeader{Header: "x-api-key", CredentialUri: "ate-secret://k8s/default/token"}, )} - h := New(client, nil, 0, nil, "") + h := New(client, nil, 0, nil) res, err := h.HandleRequestHeaders(context.Background(), withAuthorization(innerMetadata(leg, "GET", "api.example.com", nil))) @@ -347,7 +415,7 @@ func TestActorJWTInjection(t *testing.T) { func TestActorJWTInjectionSkippedWithoutHeader(t *testing.T) { client := &egressMockClient{actor: runningActor(), policy: replaceHeadersPolicy(extproc.EgressTLSMITMFilterChainName, actorJWTHeader("authorization"))} - h := New(client, nil, 0, nil, "") + h := New(client, nil, 0, nil) res, err := h.HandleRequestHeaders(context.Background(), innerMetadata(extproc.EgressTLSMITMFilterChainName, "GET", "api.example.com", nil)) if err != nil { @@ -385,7 +453,7 @@ func TestActorJWTInjectionDenials(t *testing.T) { policy: replaceHeadersPolicy(leg, actorJWTHeader(header)), mintErr: tc.mintErr, } - h := New(mock, nil, 0, nil, "") + h := New(mock, nil, 0, nil) md := innerMetadata(leg, "GET", "api.example.com", nil) md.Headers[header] = "placeholder" _, err := h.HandleRequestHeaders(context.Background(), md) diff --git a/cmd/atenet/internal/router/egress/egress.go b/cmd/atenet/internal/router/egress/egress.go index 9e8de155ff..7c0edd34b1 100644 --- a/cmd/atenet/internal/router/egress/egress.go +++ b/cmd/atenet/internal/router/egress/egress.go @@ -85,13 +85,12 @@ type Handler struct { policies *policyCache // actorJWTs mints the actor JWTs that credential injections ask for. actorJWTs *actorjwt.Minter - // provider resolves an egress policy's credential_uri injections. Nil - // means none are configured, and a request that needs one is denied. - provider credprovider.CredentialProviderClient - // providerName, when set, is the provider this gateway serves (the host of - // its ate-secret:// credential URIs); a credential URI naming another - // provider is refused. - providerName string + // providers is a map of provider name (from the URI) to the credential + // provider. + // + // A special case of a map consisting of a single entry with empty string + // ("") key will match all provider URIs. + providers map[string]credprovider.CredentialProviderClient } // New builds the egress handler. loadActorIdentityRoots yields the egress @@ -100,12 +99,11 @@ type Handler struct { // the next tunnel. A nil loadActorIdentityRoots fails every CONNECT with 503. // policyCacheTTL of 0 fetches the policy on every callout. // -// provider resolves an allowed rule's credential_uri injections on the -// TLS-terminated MITM leg; nil leaves them off, so a request that needs one is -// denied. Actor JWT injections need no provider. providerName, when set, is -// the provider this gateway serves; a credential URI naming another provider -// is refused. -func New(apiClient ateapi.ControlClient, loadActorIdentityRoots func() (*x509.CertPool, error), policyCacheTTL time.Duration, provider credprovider.CredentialProviderClient, providerName string) *Handler { +// providers resolve an allowed rule's credential_uri injections on the +// TLS-terminated MITM leg; none leaves them off, so a request that needs one is +// denied. Actor JWT injections need no provider. See Handler.providers for the +// map definition. +func New(apiClient ateapi.ControlClient, loadActorIdentityRoots func() (*x509.CertPool, error), policyCacheTTL time.Duration, providers map[string]credprovider.CredentialProviderClient) *Handler { if loadActorIdentityRoots == nil { loadActorIdentityRoots = func() (*x509.CertPool, error) { return nil, errors.New("no actor-identity CA configured") @@ -116,8 +114,7 @@ func New(apiClient ateapi.ControlClient, loadActorIdentityRoots func() (*x509.Ce loadActorIdentityRoots: loadActorIdentityRoots, policies: newPolicyCache(apiClient, policyCacheTTL), actorJWTs: actorjwt.New(apiClient), - provider: provider, - providerName: providerName, + providers: providers, } } diff --git a/cmd/atenet/internal/router/egress/egress_test.go b/cmd/atenet/internal/router/egress/egress_test.go index 79d475cb83..703db9034e 100644 --- a/cmd/atenet/internal/router/egress/egress_test.go +++ b/cmd/atenet/internal/router/egress/egress_test.go @@ -167,7 +167,7 @@ func xfccHeaderDER(chain ...[]byte) string { // egressHandler builds a Handler with an allow-everything policy, so the // identity tests are about identity alone. func egressHandler(loadRoots func() (*x509.CertPool, error), actor *ateapi.Actor, err error) *Handler { - return New(&egressMockClient{actor: actor, err: err, policy: allowAllPolicy()}, loadRoots, DefaultPolicyCacheTTL, nil, "") + return New(&egressMockClient{actor: actor, err: err, policy: allowAllPolicy()}, loadRoots, DefaultPolicyCacheTTL, nil) } // egressMockClient is the slice of ateapi the egress handler talks to. @@ -409,7 +409,7 @@ func TestConnectLegOpensForAnyRules(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - h := New(&egressMockClient{actor: runningActor(), policy: tc.policy}, ca.loadRoots, 0, nil, "") + h := New(&egressMockClient{actor: runningActor(), policy: tc.policy}, ca.loadRoots, 0, nil) md := egressMetadata(xfccHeader(leaf)) md.Host = "93.184.216.34:443" if tc.dialed != "" { @@ -468,7 +468,7 @@ func TestConnectLegWithoutRequestLegs(t *testing.T) { leaf := ca.issueActorCert(t, "spiffe://substrate-actor.local/worker-for-actor/foo/bar", actorCertOptions{}) certificate := string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leaf.Raw})) - h := New(&egressMockClient{actor: runningActor(), policy: httpPolicy("api.example.com")}, ca.loadRoots, 0, nil, "") + h := New(&egressMockClient{actor: runningActor(), policy: httpPolicy("api.example.com")}, ca.loadRoots, 0, nil) res, err := h.HandleRequestHeaders(context.Background(), agentgatewayEgressMetadata(certificate)) if err != nil { t.Fatalf("HandleRequestHeaders() error = %v, want the tunnel to open", err) @@ -476,7 +476,7 @@ func TestConnectLegWithoutRequestLegs(t *testing.T) { if got := dialedPortOf(res); got != "80" { t.Errorf("dialed port = %q, want %q", got, "80") } - h = New(&egressMockClient{actor: runningActor()}, ca.loadRoots, 0, nil, "") + h = New(&egressMockClient{actor: runningActor()}, ca.loadRoots, 0, nil) _, err = h.HandleRequestHeaders(context.Background(), agentgatewayEgressMetadata(certificate)) wantStatus(t, err, envoy_type.StatusCode_Forbidden) } diff --git a/cmd/atenet/internal/router/egress/request_test.go b/cmd/atenet/internal/router/egress/request_test.go index 89d8527717..174bcc899e 100644 --- a/cmd/atenet/internal/router/egress/request_test.go +++ b/cmd/atenet/internal/router/egress/request_test.go @@ -66,7 +66,7 @@ func cleartextInjectionPolicy(pattern string) *ateapi.EgressPolicy { // policyHandler builds a Handler for an actor whose policy is policy (nil // means none) with the cache disabled, so each callout sees the mock as is. func policyHandler(policy *ateapi.EgressPolicy) *Handler { - return New(&egressMockClient{actor: runningActor(), policy: policy}, nil, 0, nil, "") + return New(&egressMockClient{actor: runningActor(), policy: policy}, nil, 0, nil) } // testSNI is the server name the test actor's TLS connection presented, on @@ -420,7 +420,7 @@ func TestRequestLegPolicyLookup(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - h := New(tc.client, nil, 0, nil, "") + h := New(tc.client, nil, 0, nil) _, err := h.HandleRequestHeaders(context.Background(), requestMetadata("api.example.com")) wantStatus(t, err, tc.want) }) @@ -446,7 +446,7 @@ func TestConnectLegRequiresAPolicy(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - h := New(tc.client, ca.loadRoots, DefaultPolicyCacheTTL, nil, "") + h := New(tc.client, ca.loadRoots, DefaultPolicyCacheTTL, nil) res, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) if tc.want == 0 { wantAllowed(t, res, err) @@ -503,7 +503,7 @@ func TestDenialBodyIsUniform(t *testing.T) { // A caller that gives up mid-fetch is neither a denial nor an outage. func TestCanceledCallerIsNotAPolicyFailure(t *testing.T) { client := &egressMockClient{policy: allowAllPolicy(), policyGate: make(chan struct{})} - h := New(client, nil, 0, nil, "") + h := New(client, nil, 0, nil) ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { diff --git a/cmd/atenet/internal/router/egressmanifest_test.go b/cmd/atenet/internal/router/egressmanifest_test.go index 9ce84c8a09..609a9d4a3d 100644 --- a/cmd/atenet/internal/router/egressmanifest_test.go +++ b/cmd/atenet/internal/router/egressmanifest_test.go @@ -17,6 +17,7 @@ package router import ( "regexp" "slices" + "strings" "testing" "time" @@ -247,3 +248,17 @@ func TestEgressManifestsLogPathsWithoutTheQuery(t *testing.T) { t.Fatal("no access log formats found; the manifest changed shape and this test is checking nothing") } } + +func TestEgressManifestsConnectAccessLogIncludesProtocol(t *testing.T) { + t.Parallel() + logs := chainAccessLogsByLeg(bootstrapTree(t))["connect"] + if len(logs) == 0 { + t.Fatal("no connect access log found; the manifest changed shape and this test is checking nothing") + } + for i, al := range logs { + format := str(child(child(child(al, "typed_config"), "log_format"), "text_format_source"), "inline_string") + if !strings.Contains(format, "protocol=%PROTOCOL%") { + t.Errorf("connect access_log[%d] format %q does not log protocol=%%PROTOCOL%%", i, format) + } + } +} diff --git a/cmd/atenet/internal/router/extproc/egress_websocket_test.go b/cmd/atenet/internal/router/extproc/egress_websocket_test.go index dc25406859..04596139d5 100644 --- a/cmd/atenet/internal/router/extproc/egress_websocket_test.go +++ b/cmd/atenet/internal/router/extproc/egress_websocket_test.go @@ -74,7 +74,7 @@ func TestProcessReturnsWebSocketDenialAsImmediateResponse(t *testing.T) { } stream := &websocketProcessStream{ctx: context.Background(), requests: []*extprocv3.ProcessingRequest{request}} server := extproc.NewServer(0, nil, extproc.Handlers{ - extproc.DirectionEgress: egress.New(nil, nil, 0, nil, ""), + extproc.DirectionEgress: egress.New(nil, nil, 0, nil), }) if err := server.Process(stream); err != nil { diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index cd352ec377..dfa53a1370 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -220,31 +220,34 @@ func (s *RouterServer) Run(ctx context.Context) error { // is set: dial the provider over mTLS and hand the client to the handler. // With no address the handler gets no provider, and a request matching // an injection-requiring rule is denied (see egress.applyEffects). - var provider credprovider.CredentialProviderClient - var providerName string - if s.cfg.CredentialProvider.Address != "" { - providerName, err = egress.ProviderName(s.cfg.CredentialProvider.Name) + providers := map[string]credprovider.CredentialProviderClient{} + for i, address := range s.cfg.CredentialProvider.Addresses { + providerName, err := egress.ProviderName(s.cfg.CredentialProvider.Names[i]) if err != nil { return fmt.Errorf("--credential-provider-name: %w", err) } + var serverName string + if len(s.cfg.CredentialProvider.ServerNames) > 0 { + serverName = s.cfg.CredentialProvider.ServerNames[i] + } providerConn, err := egress.DialProvider(ctx, egress.ProviderDialConfig{ - Address: s.cfg.CredentialProvider.Address, + Address: address, CAFile: s.cfg.CredentialProvider.CAFile, ClientCert: s.cfg.CredentialProvider.ClientCert, - ServerName: s.cfg.CredentialProvider.ServerName, + ServerName: serverName, Insecure: s.cfg.CredentialProvider.Insecure, }) if err != nil { return fmt.Errorf("dial credential provider: %w", err) } defer providerConn.Close() - provider = credprovider.NewCredentialProviderClient(providerConn) + providers[providerName] = credprovider.NewCredentialProviderClient(providerConn) slog.InfoContext(ctx, "egress credential injection enabled", - slog.String("provider", s.cfg.CredentialProvider.Address), - slog.String("provider_name", s.cfg.CredentialProvider.Name)) + slog.String("provider", address), + slog.String("provider_name", providerName)) } - egressHandler := egress.New(s.apiClient, loadActorIdentityRoots, s.cfg.EgressPolicyCacheTTL, provider, providerName) + egressHandler := egress.New(s.apiClient, loadActorIdentityRoots, s.cfg.EgressPolicyCacheTTL, providers) handlers[egressHandler.Direction()] = egressHandler } diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index 8f9e798eda..a72ad275fa 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -385,8 +385,8 @@ func (x *XdsServer) SetTlsConfig(httpsPort int, certPath string) { // SetDownstreamClientAuth makes every TLS ingress listener require a client // certificate signed by a CA in caPath whose URI SAN exactly matches one of // allowedSPIFFEIDs. Envoy enforces this during the handshake, so a rejected -// client never reaches ext_proc. UpdateSnapshot fails if a TLS listener is -// enabled without it. +// client never reaches ext_proc. UpdateSnapshot serves no TLS listener when +// allowedSPIFFEIDs is empty, and fails if one is enabled without caPath. func (x *XdsServer) SetDownstreamClientAuth(caPath string, allowedSPIFFEIDs []string) { x.mu.Lock() defer x.mu.Unlock() @@ -520,12 +520,15 @@ func (x *XdsServer) UpdateSnapshot() error { x.mu.Lock() defer x.mu.Unlock() - httpsEnabled := x.httpsPort > 0 && x.certPath != "" + // Without an allowlist no client may reach actors, so the TLS listeners + // are not served. Envoy skips the SAN check when there are no matchers. + clientsAllowed := len(x.downstreamAllowedSPIFFEIDs) > 0 + httpsEnabled := x.httpsPort > 0 && x.certPath != "" && clientsAllowed // The main_internal cluster and listener only exist to serve the CONNECT // listener. - connectEnabled := x.connectTLSPort > 0 && x.certPath != "" - if (httpsEnabled || connectEnabled) && (x.downstreamClientCAPath == "" || len(x.downstreamAllowedSPIFFEIDs) == 0) { - return fmt.Errorf("TLS ingress listeners need client authentication: call SetDownstreamClientAuth") + connectEnabled := x.connectTLSPort > 0 && x.certPath != "" && clientsAllowed + if (httpsEnabled || connectEnabled) && x.downstreamClientCAPath == "" { + return fmt.Errorf("TLS ingress listeners need a client CA: call SetDownstreamClientAuth") } x.versionCount++ @@ -551,9 +554,10 @@ func (x *XdsServer) UpdateSnapshot() error { clusters = append(clusters, x.buildOtlpCollectorCluster()) } - // Routes - routes := []types.Resource{ - x.buildRoutes(), + // Routes. Only the HTTPS and main_internal listeners reference them. + var routes []types.Resource + if httpsEnabled || connectEnabled { + routes = append(routes, x.buildRoutes()) } // Listeners diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 1bcfec6a3d..e616ed70b4 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -1652,23 +1652,39 @@ func TestXdsServer_UpdateSnapshot_DownstreamClientAuth(t *testing.T) { } // A TLS listener without client authentication would let any client reach any -// actor, so UpdateSnapshot refuses to build one. +// actor. With no allowlist UpdateSnapshot serves no TLS listener, and with an +// allowlist but no CA it refuses to build one. func TestXdsServer_UpdateSnapshot_TLSListenerWithoutClientAuth(t *testing.T) { for _, tc := range []struct { name string httpsPort int connectTLSPort int + caPath string + allowed []string + wantErr bool }{ - {name: "HTTPS", httpsPort: 8443}, - {name: "CONNECT-TLS", connectTLSPort: 8444}, + {name: "HTTPS without allowlist", httpsPort: 8443, caPath: "/run/podidentity.podcert.ate.dev/trust-bundle.pem"}, + {name: "CONNECT-TLS without allowlist", connectTLSPort: 8444, caPath: "/run/podidentity.podcert.ate.dev/trust-bundle.pem"}, + {name: "HTTPS without CA", httpsPort: 8443, allowed: []string{"spiffe://cluster.local/ns/demo/sa/client"}, wantErr: true}, + {name: "CONNECT-TLS without CA", connectTLSPort: 8444, allowed: []string{"spiffe://cluster.local/ns/demo/sa/client"}, wantErr: true}, } { t.Run(tc.name, func(t *testing.T) { server := NewXdsServer(18000) server.SetConfig(50053, "127.0.0.1") server.SetConnectTLSPort(tc.connectTLSPort) server.SetTlsConfig(tc.httpsPort, "/run/servicedns.podcert.ate.dev/credential-bundle.pem") - if err := server.UpdateSnapshot(); err == nil { - t.Fatal("UpdateSnapshot succeeded, want an error") + server.SetDownstreamClientAuth(tc.caPath, tc.allowed) + if tc.wantErr { + if err := server.UpdateSnapshot(); err == nil { + t.Fatal("UpdateSnapshot succeeded, want an error") + } + return + } + listeners := snapshotListeners(t, server) + for _, name := range []string{IngressHTTPSListener, "connect_terminate_tls"} { + if _, exists := listeners[name]; exists { + t.Errorf("TLS listener '%s' is served without an allowlist", name) + } } }) } diff --git a/cmd/ateworker-gvisor/ckptcgroup.go b/cmd/ateworker-gvisor/ckptcgroup.go index f2a5a6055f..51c3a0c99f 100644 --- a/cmd/ateworker-gvisor/ckptcgroup.go +++ b/cmd/ateworker-gvisor/ckptcgroup.go @@ -62,8 +62,9 @@ func checkpointCgroupDir(root, actorUID string) string { // isolateCheckpointCache moves the actor's sentry from its sandbox leaf into a // sibling -_ckpt cgroup, preserving the sandbox leaf's cpu.max so checkpoint -// serialization remains capped to the actor's CPU limit. -func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actorUID string) (*checkpointCgroup, error) { +// serialization remains capped to the actor's CPU limit. cacheHigh, when set, +// is the sibling's memory.high. +func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actorUID string, cacheHigh int64) (*checkpointCgroup, error) { pid, err := readSandboxPID(actorDirs) if err != nil { return nil, err @@ -82,6 +83,14 @@ func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actor slog.Debug("Failed to copy cpu.max to checkpoint cgroup", slog.String("cgroup", dir), slog.Any("err", err)) } } + // written even when unset, so a leftover cgroup does not keep an old limit + high := "max" + if cacheHigh > 0 { + high = strconv.FormatInt(cacheHigh, 10) + } + if err := writeExisting(filepath.Join(dir, "memory.high"), high); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("while limiting cgroup %q: %w", dir, err) + } c := &checkpointCgroup{leaf: leaf, dir: dir, pid: pid} if err := movePID(dir, pid); err != nil { return nil, errors.Join(err, c.remove()) @@ -89,6 +98,17 @@ func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actor return c, nil } +// writeExisting writes a cgroup interface file without creating it, so a node +// that did not delegate the controller has nothing written. +func writeExisting(path, value string) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0) + if err != nil { + return err + } + _, err = f.WriteString(value) + return errors.Join(err, f.Close()) +} + func readSandboxPID(actorDirs *ateworkerpb.ActorDirs) (int, error) { pidFile := pidFilePath(actorDirs, ocispec.SandboxID) b, err := os.ReadFile(pidFile) @@ -156,11 +176,12 @@ func (c *checkpointCgroup) remove() error { func checkpointIsolated( ctx context.Context, cgroupRoot, actorUID string, + cacheHigh int64, actorDirs *ateworkerpb.ActorDirs, checkpoint func(context.Context) error, terminate func() error, ) error { - ckptCgroup, err := isolateCheckpointCache(cgroupRoot, actorDirs, actorUID) + ckptCgroup, err := isolateCheckpointCache(cgroupRoot, actorDirs, actorUID, cacheHigh) if err != nil { slog.WarnContext(ctx, "Checkpointing without page cache isolation", slog.String("actorUID", actorUID), slog.Any("err", err)) diff --git a/cmd/ateworker-gvisor/ckptcgroup_test.go b/cmd/ateworker-gvisor/ckptcgroup_test.go index 910c98987e..ae2127f91a 100644 --- a/cmd/ateworker-gvisor/ckptcgroup_test.go +++ b/cmd/ateworker-gvisor/ckptcgroup_test.go @@ -106,7 +106,7 @@ func mustRead(t *testing.T, path string) string { func TestIsolateCheckpointCacheMovesOnlyTheSentry(t *testing.T) { f := newCkptFixture(t, "101") - c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID) + c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 0) if err != nil { t.Fatalf("isolateCheckpointCache: %v", err) } @@ -135,7 +135,7 @@ func TestIsolateCheckpointCacheMovesOnlyTheSentry(t *testing.T) { func TestIsolateCheckpointCacheWithoutPIDFile(t *testing.T) { f := newCkptFixture(t, "") - if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID); err == nil { + if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 0); err == nil { t.Fatal("isolateCheckpointCache succeeded with no sandbox pid file") } if _, err := os.Stat(checkpointCgroupDir(f.root, testActorUID)); !os.IsNotExist(err) { @@ -147,7 +147,7 @@ func TestIsolateCheckpointCacheRejectsPIDOutsideSandboxLeaf(t *testing.T) { f := newCkptFixture(t, "101") mustWrite(t, filepath.Join(f.leaf, "cgroup.procs"), "100\n") - if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID); err == nil { + if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 0); err == nil { t.Fatal("isolateCheckpointCache succeeded with the pid outside the sandbox leaf") } if _, err := os.Stat(checkpointCgroupDir(f.root, testActorUID)); !os.IsNotExist(err) { @@ -160,7 +160,7 @@ func TestIsolateCheckpointCacheRejectsPIDOutsideSandboxLeaf(t *testing.T) { func TestCheckpointCgroupRestoreAfterLeafGone(t *testing.T) { f := newCkptFixture(t, "101") - c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID) + c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 0) if err != nil { t.Fatalf("isolateCheckpointCache: %v", err) } @@ -179,7 +179,7 @@ func TestCheckpointCgroupRestoreAfterLeafGone(t *testing.T) { func TestCheckpointCgroupRestoreSkipsExitedSentry(t *testing.T) { f := newCkptFixture(t, "101") - c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID) + c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 0) if err != nil { t.Fatalf("isolateCheckpointCache: %v", err) } @@ -221,7 +221,7 @@ func TestCheckpointIsolated(t *testing.T) { err := checkpointIsolated( context.Background(), - f.root, testActorUID, f.actorDirs, + f.root, testActorUID, 0, f.actorDirs, func(context.Context) error { if got := mustRead(t, filepath.Join(ckptDir, "cgroup.procs")); got != "101" { t.Errorf("checkpoint cgroup.procs during checkpoint = %q, want 101", got) @@ -253,7 +253,7 @@ func TestCheckpointIsolated(t *testing.T) { err := checkpointIsolated( context.Background(), - f.root, testActorUID, f.actorDirs, + f.root, testActorUID, 0, f.actorDirs, func(context.Context) error { if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { t.Errorf("checkpoint cgroup should not be created without a sandbox pid: %v", err) @@ -280,7 +280,7 @@ func TestCheckpointIsolated(t *testing.T) { err := checkpointIsolated( context.Background(), - f.root, testActorUID, f.actorDirs, + f.root, testActorUID, 0, f.actorDirs, func(context.Context) error { return errors.New("checkpoint boom") }, @@ -310,7 +310,7 @@ func TestCheckpointIsolated(t *testing.T) { err := checkpointIsolated( context.Background(), - f.root, testActorUID, f.actorDirs, + f.root, testActorUID, 0, f.actorDirs, func(context.Context) error { movePID = func(string, int) error { return unix.EACCES } return errors.New("checkpoint boom") @@ -339,7 +339,7 @@ func TestCheckpointIsolated(t *testing.T) { err := checkpointIsolated( context.Background(), - f.root, testActorUID, f.actorDirs, + f.root, testActorUID, 0, f.actorDirs, func(context.Context) error { if err := os.RemoveAll(f.leaf); err != nil { t.Fatal(err) @@ -369,7 +369,7 @@ func TestCheckpointIsolated(t *testing.T) { err := checkpointIsolated( context.Background(), - f.root, testActorUID, f.actorDirs, + f.root, testActorUID, 0, f.actorDirs, func(context.Context) error { return nil }, func() error { // Non-empty ckptDir emulates a sentry still alive in the cgroup. @@ -616,3 +616,24 @@ func TestEvictPreservedRestoreDir(t *testing.T) { } }) } + +func TestIsolateCheckpointCacheLimitsTheCache(t *testing.T) { + f := newCkptFixture(t, "101") + // cgroupfs provides memory.high itself + dir := checkpointCgroupDir(f.root, testActorUID) + mustMkdir(t, dir) + mustWrite(t, filepath.Join(dir, "memory.high"), "max\n") + if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 64<<20); err != nil { + t.Fatalf("isolateCheckpointCache: %v", err) + } + if got := mustRead(t, filepath.Join(dir, "memory.high")); got != "67108864" { + t.Errorf("checkpoint memory.high = %q, want 67108864", got) + } + // a leftover cgroup reused without a limit drops the old one + if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 0); err != nil { + t.Fatalf("isolateCheckpointCache: %v", err) + } + if got := mustRead(t, filepath.Join(dir, "memory.high")); got != "max" { + t.Errorf("checkpoint memory.high = %q, want max", got) + } +} diff --git a/cmd/ateworker-gvisor/crash.go b/cmd/ateworker-gvisor/crash.go index e6e1b83cb8..e124217a6f 100644 --- a/cmd/ateworker-gvisor/crash.go +++ b/cmd/ateworker-gvisor/crash.go @@ -58,7 +58,7 @@ func (s *AteWorkerService) watchSandbox(ctx context.Context, hosted *hostedActor } ctx := context.Background() message := "gVisor sandbox exited" - if hosted.oomKilled.Load() { + if hosted.memory.Killed() { message = "killed for exceeding its memory limit" } diff --git a/cmd/ateworker-gvisor/hosted.go b/cmd/ateworker-gvisor/hosted.go index 47e9b56e1d..749860b342 100644 --- a/cmd/ateworker-gvisor/hosted.go +++ b/cmd/ateworker-gvisor/hosted.go @@ -21,7 +21,6 @@ import ( "fmt" "log/slog" "net" - "sync" "sync/atomic" "time" @@ -55,20 +54,15 @@ type hostedActor struct { // nothing that waits on it later can be fooled by a reused pid. Nil if it // could not be pinned. Set once, under the actor's lock. sandbox *sandboxProcess - // memoryLimit is the sandbox leaf's memory.high, or 0 while watchMemory - // should leave it alone. - memoryLimit atomic.Uint64 - // oomKilled is set before watchMemory kills the sandbox. - oomKilled atomic.Bool - // memoryMu is held while watchMemory checks or kills the sandbox, and - // guards memoryWatch. - memoryMu sync.Mutex - memoryWatch cgroup.MemoryWatch + // memoryBytes is the actor's memory limit, what it books on the worker. + memoryBytes int64 + // memory kills the sandbox once it is stuck at its memory.high. + memory cgroup.MemoryGuard } // admitActor reserves capacity before network setup. An actor that is already // hosted keeps its slot; its old network is returned for the caller to close. -func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution) (*hostedActor, *actornet.SandboxSession, error) { +func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution, memoryBytes int64) (*hostedActor, *actornet.SandboxSession, error) { s.actorsMu.Lock() defer s.actorsMu.Unlock() var stale *actornet.SandboxSession @@ -87,16 +81,17 @@ func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution) (* // The sandbox cgroup is new with every activation, so its counters start // at zero. ctx, cancel := context.WithCancel(context.Background()) - hosted := &hostedActor{attribution: attribution, usage: workerstats.NewActivation(time.Now(), false), ctx: ctx, cancel: cancel} + hosted := &hostedActor{attribution: attribution, usage: workerstats.NewActivation(time.Now(), false), ctx: ctx, cancel: cancel, memoryBytes: memoryBytes} s.actors[attribution.UID] = hosted return hosted, stale, nil } -// hostActor sets up the actor's network, replacing any stale one. -func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources.ActorAttribution, actorDirs *ateworkerpb.ActorDirs) (*hostedActor, error) { +// hostActor sets up the actor's network, replacing any stale one. memoryBytes +// is what the actor books. +func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources.ActorAttribution, actorDirs *ateworkerpb.ActorDirs, memoryBytes int64) (*hostedActor, error) { uid := attribution.UID - hosted, stale, err := s.admitActor(attribution) + hosted, stale, err := s.admitActor(attribution, memoryBytes) if err != nil { return nil, err } diff --git a/cmd/ateworker-gvisor/hosted_test.go b/cmd/ateworker-gvisor/hosted_test.go index e3aebb6594..646fd62132 100644 --- a/cmd/ateworker-gvisor/hosted_test.go +++ b/cmd/ateworker-gvisor/hosted_test.go @@ -43,7 +43,7 @@ func TestAdmitActorEnforcesTheCeilingConcurrently(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - _, _, _ = s.admitActor(resources.ActorAttribution{UID: fmt.Sprintf("actor-%d", i)}) + _, _, _ = s.admitActor(resources.ActorAttribution{UID: fmt.Sprintf("actor-%d", i)}, 0) }() } wg.Wait() @@ -57,11 +57,11 @@ func TestAdmitActorEnforcesTheCeilingConcurrently(t *testing.T) { // on a full worker and a retry cannot lose its place to another actor. func TestAdmitActorKeepsAHostedActorsSlot(t *testing.T) { s := &AteWorkerService{actors: map[string]*hostedActor{}, maxActors: 1} - first, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}) + first, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, 0) if err != nil { t.Fatal(err) } - again, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}) + again, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, 0) if err != nil { t.Fatalf("re-admitting a hosted actor on a full worker: %v", err) } @@ -69,7 +69,7 @@ func TestAdmitActorKeepsAHostedActorsSlot(t *testing.T) { t.Error("re-admission kept the old record; readers could not tell the incarnations apart") } // ResourceExhausted, so the control plane treats it as a capacity miss. - if _, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-b"}); apierror.Code(err) != codes.ResourceExhausted { + if _, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-b"}, 0); apierror.Code(err) != codes.ResourceExhausted { t.Errorf("admitting past the ceiling: got %v, want ResourceExhausted", err) } if got := len(s.hostedActors()); got != 1 { @@ -85,7 +85,7 @@ func TestDrainingActorsStillCountAgainstTheCeiling(t *testing.T) { maxActors: 1, draining: 1, } - if _, err := s.hostActor(context.Background(), resources.ActorAttribution{UID: "actor-a"}, nil); err == nil { + if _, err := s.hostActor(context.Background(), resources.ActorAttribution{UID: "actor-a"}, nil, 0); err == nil { t.Error("admitted an actor while a draining one still held the only place") } } @@ -165,3 +165,14 @@ func TestBeginRPCRegistersBeforeCheckingForADrain(t *testing.T) { t.Errorf("a refused RPC stayed registered: %v", got) } } + +func TestUnbookedMemory(t *testing.T) { + const limit = 4 << 30 + s := &AteWorkerService{actors: map[string]*hostedActor{ + "a": {memoryBytes: 1 << 30}, + "b": {memoryBytes: 256 << 20}, + }} + if got, want := s.unbookedMemory(limit), int64(limit-memoryReservedBytes-1<<30-256<<20); got != want { + t.Errorf("unbookedMemory() = %d, want %d", got, want) + } +} diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index 14f386a144..f6460a2c9e 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -31,6 +31,7 @@ import ( "sync/atomic" "syscall" "time" + "unsafe" "cloud.google.com/go/compute/metadata" "github.com/spf13/pflag" @@ -43,11 +44,13 @@ import ( "go.ate.dev/substrate/internal/ateworker" "go.ate.dev/substrate/internal/ateworker/actornet" "go.ate.dev/substrate/internal/ateworker/cgroup" + "go.ate.dev/substrate/internal/ateworker/memborrow" "go.ate.dev/substrate/internal/ateworker/phaselog" "go.ate.dev/substrate/internal/ateworker/pid1" "go.ate.dev/substrate/internal/ateworker/register" "go.ate.dev/substrate/internal/ateworkervalidation" "go.ate.dev/substrate/internal/atunnel" + "go.ate.dev/substrate/internal/bootid" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/ocispec" @@ -57,15 +60,38 @@ import ( "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/internal/serverboot" "go.ate.dev/substrate/internal/sizing" + "go.ate.dev/substrate/internal/snapshotcache" "go.ate.dev/substrate/internal/version" "go.ate.dev/substrate/internal/wakeupprobe" "go.ate.dev/substrate/internal/workerstats" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "golang.org/x/sys/unix" "google.golang.org/grpc" "google.golang.org/grpc/reflection" "k8s.io/apimachinery/pkg/util/validation/field" ) +func checkCoreTagsSupported() bool { + var cookie uint64 + _, _, errno := unix.Syscall6( + unix.SYS_PRCTL, + unix.PR_SCHED_CORE, + unix.PR_SCHED_CORE_GET, + 0, + unix.PR_SCHED_CORE_SCOPE_THREAD, + uintptr(unsafe.Pointer(&cookie)), + 0, + ) + switch errno { + case 0: + return true + case unix.EINVAL, unix.ENOSYS, unix.ENODEV: + return false + default: + panic(fmt.Sprintf("prctl(PR_SCHED_CORE_GET) failed: %v", errno)) + } +} + func flagsConfigPath() string { path := "internal_flags.json" if koDataPath := os.Getenv("KO_DATA_PATH"); koDataPath != "" { @@ -138,6 +164,10 @@ func do(ctx context.Context) error { if *usageSampleInterval < minUsageSampleInterval { return fmt.Errorf("--usage-sample-interval must be at least %v, got %v", minUsageSampleInterval, *usageSampleInterval) } + bootID, err := bootid.Read(bootid.Path) + if err != nil { + return err + } const serviceName = "ateworker-gvisor" // Export through atelet's node-local relay when it is there, so telemetry @@ -237,8 +267,8 @@ func do(ctx context.Context) error { if err != nil { return err } - ateworkerService := NewService(tunnel, actorLogger, *maxActors, flags) - ateworkerService.throttleMemory = psiEnabled(defaultCgroupRoot) + ateworkerService := NewService(tunnel, actorLogger, *maxActors, flags, bootID) + ateworkerService.throttleMemory = cgroup.PSIEnabled(defaultCgroupRoot) if !ateworkerService.throttleMemory { slog.WarnContext(ctx, "Kernel has no cgroup memory pressure (PSI), so sandboxes keep memory.max and an OOM kill in one can restart the worker") } @@ -293,6 +323,7 @@ func do(ctx context.Context) error { AteletSPIFFEID: tunnelConfig.BrokerIdentity, Actors: *maxActors, PodIPs: register.ParsePodIPs(*podIPs), + MemoryReservedBytes: memoryReservedBytes, SandboxClass: string(atev1.SandboxClassGvisor), RuntimesFile: nodepath.WorkerSandboxRuntimesPath(*podUID), } @@ -342,6 +373,9 @@ type AteWorkerService struct { // crashReporter reports actors whose sandbox died. Nil if it could not be // set up. crashReporter crashReporter + // bootID is the node's boot ID, recorded with each completed checkpoint so + // a retry after a reboot does not reuse it. + bootID string // Guards actors and draining against concurrent stats reads. actorsMu sync.RWMutex @@ -381,22 +415,35 @@ type AteWorkerService struct { // flags holds the watched JSON flag configuration. flags *flagStore + + // checkpointWrites holds a slot per checkpoint writing its page cache, + // which memoryReservedBytes budgets for. + checkpointWrites chan struct{} + borrow *memborrow.Ledger + + // enableCoreTags indicates whether the host kernel supports core tagging. + enableCoreTags bool } var _ ateworkerpb.WorkerServer = (*AteWorkerService)(nil) // NewService creates a new AteWorkerService. -func NewService(tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, maxActors int, flags *flagStore) *AteWorkerService { - return &AteWorkerService{ - locks: actorlock.New(), - inFlight: actorlock.NewInFlight(), - actors: map[string]*hostedActor{}, - maxActors: maxActors, - tunnel: tunnel, - actorLogger: actorLogger, - cgroupRoot: defaultCgroupRoot, - flags: flags, - } +func NewService(tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, maxActors int, flags *flagStore, bootID string) *AteWorkerService { + s := &AteWorkerService{ + locks: actorlock.New(), + inFlight: actorlock.NewInFlight(), + actors: map[string]*hostedActor{}, + maxActors: maxActors, + tunnel: tunnel, + actorLogger: actorLogger, + cgroupRoot: defaultCgroupRoot, + flags: flags, + checkpointWrites: make(chan struct{}, checkpointWriteSlots), + bootID: bootID, + enableCoreTags: checkCoreTagsSupported(), + } + s.borrow = &memborrow.Ledger{Unbooked: s.unbookedMemory} + return s } // beginRPC registers before checking the drain flag so shutdown cannot miss it. @@ -654,7 +701,7 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run return nil, err } // Publish attribution before boot so stats can include startup usage. - hosted, err := s.hostActor(ctx, attribution, req.GetActorDirs()) + hosted, err := s.hostActor(ctx, attribution, req.GetActorDirs(), sizing.FromLimits(req.GetResources().GetLimits()).MemoryBytes) if err != nil { return nil, err } @@ -666,11 +713,12 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run durableVolumes: durableVolumeNames(req.GetSpec()), containers: req.GetSpec().GetContainers(), flags: s.flags, + enableCoreTags: s.enableCoreTags, } var containersToDelete []string defer func() { if retErr != nil { - if hosted.oomKilled.Load() { + if hosted.memory.Killed() { retErr = fmt.Errorf("killed for exceeding its memory limit: %w", retErr) } cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) @@ -692,6 +740,9 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run } } }() + if err := s.borrow.Wait(ctx); err != nil { + return nil, err + } // Egress is activated before the sandbox starts, ingress is activated after // the wakeup probe (see tunnel.Tunnel.ActivateEgress). if err := s.tunnel.ActivateEgress(attribution, egress); err != nil { @@ -773,13 +824,13 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke defer s.locks.Unlock(req.GetActorUid()) if s.lookupActor(req.GetActorUid()) == nil { - if resp := ateworker.ReuseCheckpoint(req.GetActorDirs().GetCheckpointDir(), req.GetFidelity()); resp != nil { + if resp := ateworker.ReuseCheckpoint(req.GetActorDirs().GetCheckpointDir(), s.bootID, req.GetFidelity()); resp != nil { slog.InfoContext(ctx, "Reusing completed checkpoint from previous attempt", slog.String("actorUID", req.GetActorUid())) return resp, nil } } - ctx, cancel := context.WithCancel(ctx) + ctx, cancel := ateworker.CheckpointContext(ctx) defer cancel() // Not cancelable: a checkpoint is saving the actor's state. defer s.inFlight.Add(req.GetActorUid(), rpcCheckpointWorkload, nil)() @@ -815,10 +866,11 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke // Checkpoint only saves state; no sizing is applied, so size is left zero. rcmd := &runsc{ - path: req.GetRunscPath(), - actorUID: req.GetActorUid(), - actorDirs: req.GetActorDirs(), - flags: s.flags, + path: req.GetRunscPath(), + actorUID: req.GetActorUid(), + actorDirs: req.GetActorDirs(), + flags: s.flags, + enableCoreTags: s.enableCoreTags, } checkpointPath := req.GetActorDirs().GetCheckpointDir() @@ -919,15 +971,28 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke default: return nil, fmt.Errorf("unsupported snapshot fidelity: %v", fidelity) } + if s.checkpointWrites != nil { + select { + case s.checkpointWrites <- struct{}{}: + case <-ctx.Done(): + return nil, fmt.Errorf("while waiting to checkpoint: %w", ctx.Err()) + } + } // the sentry leaves its sandbox leaf for the checkpoint cgroup var memoryLimit uint64 if hosted != nil { - memoryLimit = pauseMemoryWatch(hosted) + memoryLimit = hosted.memory.Pause() + } + cacheHigh, borrowed := s.checkpointCacheLimit(hosted, req.GetActorUid()) + err = checkpointIsolated(ctx, s.cgroupRoot, req.GetActorUid(), cacheHigh, req.GetActorDirs(), checkpointFn, terminate) + if s.checkpointWrites != nil { + <-s.checkpointWrites } - if err := checkpointIsolated(ctx, s.cgroupRoot, req.GetActorUid(), req.GetActorDirs(), checkpointFn, terminate); err != nil { - if hosted != nil && memoryLimit != 0 { - resumeMemoryWatch(hosted, memoryLimit) + if err != nil { + if hosted != nil { + hosted.memory.Resume(memoryLimit) } + s.borrow.Return(borrowed) return nil, err } @@ -936,14 +1001,24 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke // rather than a hardcoded list. snapshotFiles, err := listSnapshotFiles(checkpointPath) if err != nil { + s.borrow.Return(borrowed) return nil, fmt.Errorf("while listing checkpoint files: %w", err) } + // the borrowed page cache is returned once it is clean, and reclaimable + written := make([]string, len(snapshotFiles)) + for i, f := range snapshotFiles { + written[i] = filepath.Join(checkpointPath, f) + } + if err := snapshotcache.StartWriteback(written, func() { s.borrow.Return(borrowed) }); err != nil { + s.borrow.Return(borrowed) + slog.WarnContext(ctx, "Failed to start checkpoint writeback", slog.String("actorUID", req.GetActorUid()), slog.Any("err", err)) + } durableFiles := durableSnapshotFiles(snapshotFiles) s.actorLogger.EmitLifecycleLog(ctx, "Actor checkpointed", attribution) resp := &ateworkerpb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles, DataSnapshotFiles: durableFiles} - if err := ateworker.RecordCheckpoint(checkpointPath, fidelity, resp); err != nil { + if err := ateworker.RecordCheckpoint(checkpointPath, s.bootID, fidelity, resp); err != nil { slog.WarnContext(ctx, "Failed to record checkpoint completion marker", slog.String("actorUID", req.GetActorUid()), slog.Any("err", err)) } return resp, nil @@ -1104,7 +1179,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb if err != nil { return nil, err } - hosted, err := s.hostActor(ctx, attribution, req.GetActorDirs()) + hosted, err := s.hostActor(ctx, attribution, req.GetActorDirs(), sizing.FromLimits(req.GetResources().GetLimits()).MemoryBytes) timing.netSetup = lap(&tLast) if err != nil { return nil, err @@ -1117,11 +1192,12 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb durableVolumes: durableVolumeNames(req.GetSpec()), containers: req.GetSpec().GetContainers(), flags: s.flags, + enableCoreTags: s.enableCoreTags, } var containersToDelete []string defer func() { if retErr != nil { - if hosted.oomKilled.Load() { + if hosted.memory.Killed() { retErr = fmt.Errorf("killed for exceeding its memory limit: %w", retErr) } cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) @@ -1140,6 +1216,9 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb } } }() + if err := s.borrow.Wait(ctx); err != nil { + return nil, err + } err = s.tunnel.ActivateEgress(attribution, egress) timing.activate = lap(&tLast) if err != nil { @@ -1301,10 +1380,11 @@ func (s *AteWorkerService) terminateWorkload(ctx context.Context, actorRef resou } rcmd := &runsc{ - path: runscPath, - actorUID: actorUID, - actorDirs: actorDirs, - flags: s.flags, + path: runscPath, + actorUID: actorUID, + actorDirs: actorDirs, + flags: s.flags, + enableCoreTags: s.enableCoreTags, } // Detached from the caller: a deadline mid-`runsc delete` would leave the diff --git a/cmd/ateworker-gvisor/memlimit.go b/cmd/ateworker-gvisor/memlimit.go index 3a9ed5551f..83b7f85c92 100644 --- a/cmd/ateworker-gvisor/memlimit.go +++ b/cmd/ateworker-gvisor/memlimit.go @@ -33,20 +33,11 @@ import ( "go.ate.dev/substrate/internal/ocispec" ) -// With runc 1.5.2+ on systemd 253-258 the worker container's scope has -// OOMPolicy=kill, so any kernel OOM kill under it stops the whole worker, even -// one confined to a single actor's leaf. So the sandbox leaf gets memory.high -// at the actor's limit, which throttles without invoking the OOM killer, and -// we kill a sandbox that stays over with cgroup.kill, which systemd ignores. -// https://github.com/opencontainers/cgroups/blob/19a241326c83a5100dff94732733791a0c9e4720/systemd/v2.go#L366-L382 - -// memoryWatchInterval is how often sandbox leaves are checked. -const memoryWatchInterval = 250 * time.Millisecond - // throttleSandboxMemory swaps the memory.max runsc set on the sandbox leaf for -// a memory.high at the same limit. runsc reads memory.max back to size the -// sentry's --total-memory when create launches the sandbox, so this runs after -// the sandbox's create. On failure the leaf keeps its memory.max. +// a memory.high at the same limit (see cgroup.MemoryGuard). runsc reads +// memory.max back to size the sentry's --total-memory when create launches the +// sandbox, so this runs after the sandbox's create. On failure the leaf keeps +// its memory.max. func (s *AteWorkerService) throttleSandboxMemory(ctx context.Context, hosted *hostedActor) { if !s.throttleMemory { return @@ -57,14 +48,7 @@ func (s *AteWorkerService) throttleSandboxMemory(ctx context.Context, hosted *ho slog.LogAttrs(ctx, slog.LevelWarn, "Sandbox keeps its memory.max, an OOM kill in it will restart the worker", attrs...) return } - hosted.memoryLimit.Store(limit) -} - -// psiEnabled reports whether the kernel tracks memory pressure for cgroups -// under root, which the watch needs to tell a stuck sandbox from a busy one. -func psiEnabled(root string) bool { - _, err := os.Stat(filepath.Join(root, "memory.pressure")) - return err == nil + hosted.memory.Resume(limit) } // removeStaleSandboxLeaf removes a sandbox leaf an earlier activation of the @@ -120,7 +104,7 @@ func swapMemoryMaxForHigh(dir string) (uint64, error) { // watchMemory kills sandboxes that stay over their memory limit, until ctx is // done. func (s *AteWorkerService) watchMemory(ctx context.Context) { - ticker := time.NewTicker(memoryWatchInterval) + ticker := time.NewTicker(cgroup.MemoryGuardInterval) defer ticker.Stop() for { select { @@ -134,54 +118,23 @@ func (s *AteWorkerService) watchMemory(ctx context.Context) { } } -// checkMemory kills h's sandbox if the watch finds it stuck at its limit. The -// kill runs in the background, holding h.memoryMu until the sandbox is gone. +// checkMemory kills h's sandbox if it is stuck at its memory limit. func (s *AteWorkerService) checkMemory(ctx context.Context, h *hostedActor) { - if !h.memoryMu.TryLock() { - // being killed, or paused for a checkpoint - return - } - limit := h.memoryLimit.Load() - if limit == 0 || h.stopping.Load() || h.oomKilled.Load() { - h.memoryMu.Unlock() + if h.stopping.Load() { return } - dir := s.sandboxLeaf(h.attribution.UID) - over, current, err := h.memoryWatch.Check(dir, limit, time.Now()) - if err != nil || !over { - // a read fails when the sandbox is going away - h.memoryMu.Unlock() + kill, ok := h.memory.Check(ctx, s.cgroupRoot, filepath.Base(s.sandboxLeaf(h.attribution.UID))) + if !ok { return } - h.oomKilled.Store(true) attrs := append(ateattr.ActorLogAttrs(h.attribution), - slog.Uint64("memoryLimitBytes", limit), - slog.Uint64(string(ateattr.StatsMemoryUsageKey), current)) + slog.Uint64("memoryLimitBytes", kill.Limit), + slog.Uint64(string(ateattr.StatsMemoryUsageKey), kill.Current)) slog.LogAttrs(ctx, slog.LevelWarn, "Killing a sandbox over its memory limit", attrs...) go func() { - defer h.memoryMu.Unlock() - killCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - if err := cgroup.KillLeafUnder(killCtx, s.cgroupRoot, filepath.Base(dir)); err != nil { + if err := <-kill.Done; err != nil { attrs = append(attrs, slog.Any("err", err)) slog.LogAttrs(ctx, slog.LevelError, "Failed to kill a sandbox over its memory limit", attrs...) } }() } - -// pauseMemoryWatch stops the watch on h's sandbox leaf, e.g. while its sentry -// sits in the checkpoint cgroup, waiting out a kill in progress. It returns -// the limit to resume with. -func pauseMemoryWatch(h *hostedActor) uint64 { - h.memoryMu.Lock() - defer h.memoryMu.Unlock() - return h.memoryLimit.Swap(0) -} - -// resumeMemoryWatch watches h's sandbox leaf again, from a fresh window. -func resumeMemoryWatch(h *hostedActor, limit uint64) { - h.memoryMu.Lock() - defer h.memoryMu.Unlock() - h.memoryWatch = cgroup.MemoryWatch{} - h.memoryLimit.Store(limit) -} diff --git a/cmd/ateworker-gvisor/memlimit_test.go b/cmd/ateworker-gvisor/memlimit_test.go index c6fc94a761..64f0a7e0b4 100644 --- a/cmd/ateworker-gvisor/memlimit_test.go +++ b/cmd/ateworker-gvisor/memlimit_test.go @@ -112,23 +112,23 @@ func TestCheckMemoryKillsOncePausesForCheckpoint(t *testing.T) { "cgroup.kill": "", "cgroup.events": "populated 0\nfrozen 0\n", }) - h.memoryLimit.Store(limit) + h.memory.Resume(limit) s.checkMemory(context.Background(), h) - if !h.oomKilled.Load() { + if !h.memory.Killed() { t.Fatal("sandbox over its limit was not killed") } - if got := pauseMemoryWatch(h); got != limit { - t.Errorf("pauseMemoryWatch() = %d, want %d", got, limit) + if got := h.memory.Pause(); got != limit { + t.Errorf("Pause() = %d, want %d", got, limit) } if got := readCgroupFile(t, leaf, "cgroup.kill"); got != "1" { t.Errorf("cgroup.kill = %q after pausing, want the kill done", got) } writeCgroupFiles(t, leaf, map[string]string{"cgroup.kill": ""}) - resumeMemoryWatch(h, limit) + h.memory.Resume(limit) s.checkMemory(context.Background(), h) - pauseMemoryWatch(h) + h.memory.Pause() if got := readCgroupFile(t, leaf, "cgroup.kill"); got != "" { t.Errorf("sandbox was killed twice") } diff --git a/cmd/ateworker-gvisor/memory.go b/cmd/ateworker-gvisor/memory.go new file mode 100644 index 0000000000..4f1e93deaf --- /dev/null +++ b/cmd/ateworker-gvisor/memory.go @@ -0,0 +1,72 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "go.ate.dev/substrate/internal/ateworker/cgroup" + "go.ate.dev/substrate/internal/ocispec" +) + +// checkpointWriteHeadroom is how far a checkpoint's page cache may grow when +// the actor's booking leaves less than that, and checkpointWriteSlots how many +// checkpoints may write at once. memoryReservedBytes covers both. +const ( + checkpointWriteHeadroom = 16 << 20 + checkpointWriteSlots = 2 +) + +// memoryReservedBytes is what the worker keeps from actors: its own process, +// and the page cache of the checkpoints it lets write at once, which leaves +// the actor's cgroup with its sentry (see isolateCheckpointCache). +const memoryReservedBytes = 32<<20 + checkpointWriteSlots*checkpointWriteHeadroom + +// unbookedMemory is the worker's reported memory capacity less what its +// actors booked. +func (s *AteWorkerService) unbookedMemory(podLimit int64) int64 { + unbooked := podLimit - memoryReservedBytes + s.actorsMu.RLock() + defer s.actorsMu.RUnlock() + for _, h := range s.actors { + unbooked -= h.memoryBytes + } + return unbooked +} + +// checkpointCacheLimit returns the memory.high for the actor's checkpoint +// cgroup, 0 for none, and what it borrowed for it. +// +// The checkpoint writes about as much as the sandbox holds through the page +// cache, charged to the checkpoint cgroup. At the worker pod's memory limit +// that can OOM-kill the whole pod, so the cache may only grow into what the +// actor booked and the sandbox left untouched, plus what it can borrow. When +// the pod can lend all of it there is no limit: memory.high also shrinks the +// cgroup's dirty page limit, which throttles the write to the disk. +func (s *AteWorkerService) checkpointCacheLimit(hosted *hostedActor, actorUID string) (high, borrowed int64) { + used, err := cgroup.ActorLeafMemory(ocispec.GVisorCgroupLeaf(actorUID, ocispec.SandboxID)) + if err != nil { + return checkpointWriteHeadroom, 0 + } + borrowed = s.borrow.Take(used) + if borrowed >= used { + return 0, borrowed + } + var booked int64 + if hosted != nil { + booked = hosted.memoryBytes + } + return max(checkpointWriteHeadroom, booked-used) + borrowed, borrowed +} diff --git a/cmd/ateworker-gvisor/runsc.go b/cmd/ateworker-gvisor/runsc.go index 18faacc6ec..2af7a015ed 100644 --- a/cmd/ateworker-gvisor/runsc.go +++ b/cmd/ateworker-gvisor/runsc.go @@ -46,6 +46,8 @@ type runsc struct { containers []*ateworkerpb.Container // flags holds the configured flags; must not be nil. flags *flagStore + // enableCoreTags enables Linux core scheduling tags (--enable-core-tags). + enableCoreTags bool } // buildArgs constructs the runsc argument vector for command, appending @@ -58,8 +60,11 @@ func (r *runsc) buildArgs(command, containerName string, extraArgs ...string) [] subcommandFlags := cfg.subcommandFlags(command) args := make([]string, 0, - len(globalFlags)+2+1+len(subcommandFlags)+len(extraArgs)+1) + len(globalFlags)+1+2+1+len(subcommandFlags)+len(extraArgs)+1) args = append(args, globalFlags...) + if r.enableCoreTags { + args = append(args, "--enable-core-tags") + } args = append(args, "-root", runscStateDir(r.actorDirs)) args = append(args, command) args = append(args, subcommandFlags...) diff --git a/cmd/ateworker-gvisor/runsc_test.go b/cmd/ateworker-gvisor/runsc_test.go index 16e50620ec..d96ccc9ab1 100644 --- a/cmd/ateworker-gvisor/runsc_test.go +++ b/cmd/ateworker-gvisor/runsc_test.go @@ -209,6 +209,29 @@ func TestBuildArgs(t *testing.T) { } } +func TestBuildArgsEnableCoreTags(t *testing.T) { + r := &runsc{ + path: "/usr/bin/runsc", + actorUID: "test-actor-123", + actorDirs: testActorDirs, + flags: mustFlagStore(t), + enableCoreTags: true, + } + + got := r.buildArgs("wait", "my-container") + want := []string{ + "-global1", + "-global2", + "--enable-core-tags", + "-root", "/node/actors/test-actor-123/runsc-state", + "wait", + "my-container", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("buildArgs(\"wait\", \"my-container\") = %v, want %v", got, want) + } +} + func TestCheckpointArgs(t *testing.T) { r := &runsc{ path: "/usr/bin/runsc", diff --git a/cmd/ateworker-microvm/checkpoint.go b/cmd/ateworker-microvm/checkpoint.go index 4686651f98..18aefd741e 100644 --- a/cmd/ateworker-microvm/checkpoint.go +++ b/cmd/ateworker-microvm/checkpoint.go @@ -31,10 +31,12 @@ import ( "go.ate.dev/substrate/cmd/ateworker-microvm/internal/ch" "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" "go.ate.dev/substrate/internal/ateworker" + "go.ate.dev/substrate/internal/ateworker/cgroup" "go.ate.dev/substrate/internal/ateworker/phaselog" "go.ate.dev/substrate/internal/ateworkervalidation" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/snapshotcache" "go.ate.dev/substrate/internal/workerstats" "golang.org/x/sync/errgroup" ) @@ -80,13 +82,13 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke defer s.locks.Unlock(req.GetActorUid()) if s.lookupActor(req.GetActorUid()) == nil { - if resp := ateworker.ReuseCheckpoint(req.GetActorDirs().GetCheckpointDir(), req.GetFidelity()); resp != nil { + if resp := ateworker.ReuseCheckpoint(req.GetActorDirs().GetCheckpointDir(), s.bootID, req.GetFidelity()); resp != nil { slog.InfoContext(ctx, "Reusing completed checkpoint from previous attempt", slog.String("actorUID", req.GetActorUid())) return resp, nil } } - ctx, cancel := context.WithCancel(ctx) + ctx, cancel := ateworker.CheckpointContext(ctx) defer cancel() defer s.inFlight.Add(req.GetActorUid(), rpcCheckpointWorkload, nil)() @@ -261,7 +263,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke slog.Duration("teardown", dTeardown)) s.recordFinalIfEnded(ctx, hosted) resp := &ateworkerpb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles, DataSnapshotFiles: durableFiles} - if err := ateworker.RecordCheckpoint(checkpointDir, scope, resp); err != nil { + if err := ateworker.RecordCheckpoint(checkpointDir, s.bootID, scope, resp); err != nil { slog.WarnContext(ctx, "Failed to record checkpoint completion marker", slog.String("actorUID", actorUID), slog.Any("err", err)) } return resp, nil @@ -288,8 +290,8 @@ func (s *AteWorkerService) snapshotVMState(ctx context.Context, client *ch.Clien slog.InfoContext(ctx, "Snapshotting guest", slog.String("id", actorUID), slog.String("dir", checkpointDir)) tSnapshot := time.Now() - if err := client.Snapshot(ctx, checkpointDir); err != nil { - return 0, fmt.Errorf("while snapshotting guest: %w", err) + if err := s.writeSnapshot(ctx, client, ra, actorUID, checkpointDir); err != nil { + return 0, err } dSnapshot := time.Since(tSnapshot) @@ -319,6 +321,84 @@ func (s *AteWorkerService) snapshotVMState(ctx context.Context, client *ch.Clien return dSnapshot, nil } +// snapshotWriteHeadroom is how far the actor's cgroup may grow past the paused +// guest while CH writes the snapshot, when its booking leaves less than that, +// and snapshotWriteSlots how many of those writes may run at once. +// memoryReservedBytes covers both. +const ( + snapshotWriteHeadroom = 16 << 20 + snapshotWriteSlots = 2 +) + +// writeSnapshot has CH write the paused guest into checkpointDir. +// +// CH writes through the page cache from the actor's cgroup while the guest +// still holds its memory. At the worker pod's memory limit that can OOM-kill +// the whole pod, and every actor on it, so the leaf may only grow into what the +// actor booked and the guest left untouched, plus what it can borrow, and the +// kernel throttles CH to the disk past that. +func (s *AteWorkerService) writeSnapshot(ctx context.Context, client *ch.Client, ra *runningActor, actorUID, checkpointDir string) error { + if s.snapshotWrites != nil { + select { + case s.snapshotWrites <- struct{}{}: + defer func() { <-s.snapshotWrites }() + case <-ctx.Done(): + return fmt.Errorf("while waiting to snapshot: %w", ctx.Err()) + } + } + // the page cache the write adds isn't the actor running away + if ra != nil { + defer ra.memory.Resume(ra.memory.Pause()) + } + var borrowed int64 + // the write caches about as much as the guest has populated, which is + // at most the leaf's usage. When the pod can lend all of that, the leaf + // goes unlimited: memory.high also shrinks the cgroup's dirty page + // limit, which throttles the write to the disk long before the cap + current, err := cgroup.ActorLeafMemory(actorUID) + if err == nil { + borrowed = s.borrow.Take(current) + } + if err == nil && borrowed >= current && s.throttleMemory { + release, err := cgroup.LiftActorLeafMemory(actorUID) + if err != nil { + slog.WarnContext(ctx, "Snapshotting under the actor's memory limit", slog.String("id", actorUID), slog.Any("err", err)) + } else { + defer release() + } + } + if err != nil || borrowed < current { + var ceiling int64 + if ra != nil { + ceiling = ra.memoryBytes + } + release, err := cgroup.LimitActorLeafGrowth(actorUID, snapshotWriteHeadroom, ceiling, borrowed) + if err != nil { + slog.WarnContext(ctx, "Snapshotting without a page cache limit", slog.String("id", actorUID), slog.Any("err", err)) + } else { + defer release() + } + } + if err := client.Snapshot(ctx, checkpointDir); err != nil { + s.borrow.Return(borrowed) + return fmt.Errorf("while snapshotting guest: %w", err) + } + // clean pages can be reclaimed, and dropped once a restore has read them. + // the tars sync themselves + var written []string + for _, name := range []string{"config.json", "state.json", memoryRangesFile} { + p := filepath.Join(checkpointDir, name) + if _, err := os.Stat(p); err == nil { + written = append(written, p) + } + } + if err := snapshotcache.StartWriteback(written, func() { s.borrow.Return(borrowed) }); err != nil { + s.borrow.Return(borrowed) + slog.WarnContext(ctx, "Failed to start snapshot writeback", slog.String("id", actorUID), slog.Any("err", err)) + } + return nil +} + // mergeOnDemandDelta merges the OnDemand delta in checkpointDir with the base in // restoreSourceDir. A preserved base is copied, never modified in place. func mergeOnDemandDelta(ctx context.Context, restoreSourceDir, checkpointDir string, preserveRestoreSource bool) error { diff --git a/cmd/ateworker-microvm/crash.go b/cmd/ateworker-microvm/crash.go index d27c589ee8..b7eea80a8a 100644 --- a/cmd/ateworker-microvm/crash.go +++ b/cmd/ateworker-microvm/crash.go @@ -53,6 +53,9 @@ func (s *AteWorkerService) watchVM(attribution resources.ActorAttribution, actor } ctx := context.Background() message := fmt.Sprintf("%s exited: %s", what, exit) + if ra.memory.Killed() { + message = "killed for exceeding its memory limit" + } slog.WarnContext(ctx, "Micro-VM died", slog.String("actorUID", attribution.UID), slog.String("reason", message)) // Not reported under the actor's lock: a suspend in flight holds the diff --git a/cmd/ateworker-microvm/hosted.go b/cmd/ateworker-microvm/hosted.go index 9b2fe6c69c..ec07b290c7 100644 --- a/cmd/ateworker-microvm/hosted.go +++ b/cmd/ateworker-microvm/hosted.go @@ -48,13 +48,16 @@ type hostedActor struct { // usage is this activation's epoch, CPU baseline, and latest sample. // Immutable after admission. usage *workerstats.Activation + // memoryBytes is the actor's memory limit, what it books on the worker. + // Immutable after admission. + memoryBytes int64 } // admitActor reserves capacity before network setup. An actor that is already // hosted keeps its slot; its old network is returned for the caller to close. // resumesGuest is true when the activation resumes a guest from a snapshot, -// whose CPU counters come back with it. -func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution, resumesGuest bool) (*hostedActor, *actornet.SandboxSession, error) { +// whose CPU counters come back with it. memoryBytes is what the actor books. +func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution, resumesGuest bool, memoryBytes int64) (*hostedActor, *actornet.SandboxSession, error) { s.actorsMu.Lock() defer s.actorsMu.Unlock() var stale *actornet.SandboxSession @@ -63,17 +66,17 @@ func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution, re } else if len(s.actors)+s.draining >= s.maxActors { return nil, nil, apierror.ResourceExhausted("worker is full: %d actors", s.maxActors) } - hosted := &hostedActor{attribution: attribution, usage: workerstats.NewActivation(time.Now(), resumesGuest)} + hosted := &hostedActor{attribution: attribution, usage: workerstats.NewActivation(time.Now(), resumesGuest), memoryBytes: memoryBytes} s.actors[attribution.UID] = hosted return hosted, stale, nil } // hostActor sets up the actor's network, replacing any stale one. resumesGuest -// is as for admitActor. -func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources.ActorAttribution, resumesGuest bool) (*hostedActor, error) { +// and memoryBytes are as for admitActor. +func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources.ActorAttribution, resumesGuest bool, memoryBytes int64) (*hostedActor, error) { uid := attribution.UID - hosted, stale, err := s.admitActor(attribution, resumesGuest) + hosted, stale, err := s.admitActor(attribution, resumesGuest, memoryBytes) if err != nil { return nil, err } diff --git a/cmd/ateworker-microvm/hosted_test.go b/cmd/ateworker-microvm/hosted_test.go index 1ce50ec51b..0a6b0039e6 100644 --- a/cmd/ateworker-microvm/hosted_test.go +++ b/cmd/ateworker-microvm/hosted_test.go @@ -29,11 +29,11 @@ import ( // on a full worker and a retry cannot lose its place to another actor. func TestAdmitActorKeepsAHostedActorsSlot(t *testing.T) { s := &AteWorkerService{actors: map[string]*hostedActor{}, maxActors: 1} - first, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, false) + first, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, false, 0) if err != nil { t.Fatal(err) } - again, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, false) + again, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, false, 0) if err != nil { t.Fatalf("re-admitting a hosted actor on a full worker: %v", err) } @@ -41,7 +41,7 @@ func TestAdmitActorKeepsAHostedActorsSlot(t *testing.T) { t.Error("re-admission kept the old record; readers could not tell the incarnations apart") } // ResourceExhausted, so the control plane treats it as a capacity miss. - if _, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-b"}, false); apierror.Code(err) != codes.ResourceExhausted { + if _, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-b"}, false, 0); apierror.Code(err) != codes.ResourceExhausted { t.Errorf("admitting past the ceiling: got %v, want ResourceExhausted", err) } if got := len(s.hostedActors()); got != 1 { diff --git a/cmd/ateworker-microvm/internal/kata/specconv.go b/cmd/ateworker-microvm/internal/kata/specconv.go index 5f143bb8a6..891bf07878 100644 --- a/cmd/ateworker-microvm/internal/kata/specconv.go +++ b/cmd/ateworker-microvm/internal/kata/specconv.go @@ -93,9 +93,10 @@ func SpecToAgentPB(s *specs.Spec) *agentpb.Spec { CgroupsPath: s.Linux.CgroupsPath, MaskedPaths: s.Linux.MaskedPaths, ReadonlyPaths: s.Linux.ReadonlyPaths, + Sysctl: s.Linux.Sysctl, } // TODO: forward the remaining OCI security knobs the kata-agent supports - // for parity with the OCI spec — Linux.Seccomp and Linux.Sysctl here, and + // for parity with the OCI spec — Linux.Seccomp here, and // Process.ApparmorProfile / Process.SelinuxLabel above. The MVP runs the // actor with kata's defaults for these. for _, ns := range s.Linux.Namespaces { diff --git a/cmd/ateworker-microvm/internal/kata/specconv_linux_test.go b/cmd/ateworker-microvm/internal/kata/specconv_linux_test.go index 71d321c248..4b7d3afdc7 100644 --- a/cmd/ateworker-microvm/internal/kata/specconv_linux_test.go +++ b/cmd/ateworker-microvm/internal/kata/specconv_linux_test.go @@ -162,3 +162,15 @@ func TestSpecToAgentPB_NonPositiveCPUValuesAreDropped(t *testing.T) { }) } } + +// The agent applies sysctls from the spec it is sent; a dropped map leaves +// the guest at its defaults. +func TestSpecToAgentPB_ForwardsSysctl(t *testing.T) { + const key = "net.ipv4.ip_unprivileged_port_start" + got := SpecToAgentPB(&specs.Spec{ + Linux: &specs.Linux{Sysctl: map[string]string{key: "0"}}, + }) + if got.Linux == nil || got.Linux.Sysctl[key] != "0" { + t.Fatalf("Linux.Sysctl = %v, want %s=0 carried through", got.GetLinux().GetSysctl(), key) + } +} diff --git a/cmd/ateworker-microvm/main.go b/cmd/ateworker-microvm/main.go index 9f8aac1783..33ce2d846b 100644 --- a/cmd/ateworker-microvm/main.go +++ b/cmd/ateworker-microvm/main.go @@ -45,9 +45,11 @@ import ( "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/ateworker/actornet" "go.ate.dev/substrate/internal/ateworker/cgroup" + "go.ate.dev/substrate/internal/ateworker/memborrow" "go.ate.dev/substrate/internal/ateworker/pid1" "go.ate.dev/substrate/internal/ateworker/register" "go.ate.dev/substrate/internal/atunnel" + "go.ate.dev/substrate/internal/bootid" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/otlprelay" "go.ate.dev/substrate/internal/preview" @@ -64,14 +66,13 @@ import ( ) var ( - podUID = pflag.String("pod-uid", "", "The UID of the current pod") - podIPs = pflag.String("pod-ips", "", "The comma-separated IPs of the current pod") - chBinary = pflag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).") - guestDebug = pflag.Bool("guest-debug", false, "enable guest debugging (agent debug logs, debug console, and early serial console)") - vmmMemReserve = pflag.Int("vmm-mem-reserve-mib", vmmMemReserveMiB, "Guest RAM (MiB) held back from the pod's memory limit for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the pod cgroup alongside the guest RAM. Prevents the pod OOMing when the VM is sized to the pod's memory limit.") - showVersion = pflag.Bool("version", false, "Print version and exit.") - logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") - previewFlags = pflag.StringSlice("preview", nil, "Preview gates to enable.") + podUID = pflag.String("pod-uid", "", "The UID of the current pod") + podIPs = pflag.String("pod-ips", "", "The comma-separated IPs of the current pod") + chBinary = pflag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).") + guestDebug = pflag.Bool("guest-debug", false, "enable guest debugging (agent debug logs, debug console, and early serial console)") + showVersion = pflag.Bool("version", false, "Print version and exit.") + logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") + previewFlags = pflag.StringSlice("preview", nil, "Preview gates to enable.") otlpRelaySocket = pflag.String("otlp-relay-socket", nodepath.AteletOTLPSocketPath(), "Unix socket of atelet's OTLP relay to export telemetry through, keeping it off the pod network. Empty, or absent at startup, exports directly to OTEL_EXPORTER_OTLP_ENDPOINT instead.") @@ -124,6 +125,10 @@ func do(ctx context.Context) error { if *usageSampleInterval < minUsageSampleInterval { return fmt.Errorf("--usage-sample-interval must be at least %v, got %v", minUsageSampleInterval, *usageSampleInterval) } + bootID, err := bootid.Read(bootid.Path) + if err != nil { + return err + } const serviceName = "ateworker-microvm" // Export through atelet's node-local relay when it is there, so telemetry @@ -233,7 +238,11 @@ func do(ctx context.Context) error { if err != nil { return err } - ateworkerService := NewService(*podUID, *chBinary, *guestDebug, *vmmMemReserve, *maxActors, tunnel, actorLogger) + ateworkerService := NewService(*podUID, *chBinary, *guestDebug, *maxActors, tunnel, actorLogger, bootID) + ateworkerService.throttleMemory = cgroup.PSIEnabled(cgroup.Root) + if !ateworkerService.throttleMemory { + slog.WarnContext(ctx, "Kernel has no cgroup memory pressure (PSI), so actor cgroups get no memory limit and an actor over its booking can restart the worker") + } // The controller sets both from the downward API. pool := workerstats.Pool{Namespace: os.Getenv("POD_NAMESPACE"), Name: os.Getenv("WORKER_POOL_NAME")} if pool.Namespace == "" || pool.Name == "" { @@ -243,6 +252,9 @@ func do(ctx context.Context) error { defer usageStdout.Close() ateworkerService.usage = workerstats.NewUsageEmitter(lp, usageStdout, pool) defer workerstats.StartSampler(ctx, *usageSampleInterval, func(ctx context.Context) { ateworkerService.sweepUsage(ctx) })() + if ateworkerService.throttleMemory { + go ateworkerService.watchMemory(ctx) + } svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -286,6 +298,8 @@ func do(ctx context.Context) error { AteletSPIFFEID: tunnelConfig.BrokerIdentity, Actors: *maxActors, PodIPs: register.ParsePodIPs(*podIPs), + MemoryReservedBytes: memoryReservedBytes, + ActorOverhead: actorOverhead, SandboxClass: string(atev1.SandboxClassMicroVM), RuntimesFile: nodepath.WorkerSandboxRuntimesPath(*podUID), } @@ -355,6 +369,9 @@ type AteWorkerService struct { // crashReporter reports actors whose micro-VM died. Nil if it could not // be set up. crashReporter crashReporter + // bootID is the node's boot ID, recorded with each completed checkpoint so + // a retry after a reboot does not reuse it. + bootID string // shuttingDown is set once SIGTERM has been received. While true, new workload // RPCs are rejected with codes.Unavailable so the control plane reschedules. @@ -372,10 +389,13 @@ type AteWorkerService struct { chBinary string guestDebug bool - // memReserveMiB is guest RAM (MiB) held back from the pod's memory limit for - // the cloud-hypervisor VMM + virtiofsd (host processes sharing the pod cgroup - // with the guest RAM). Set from --vmm-mem-reserve-mib. - memReserveMiB int + // snapshotWrites holds a slot per CH snapshot write in flight, whose page + // cache memoryReservedBytes budgets for. + snapshotWrites chan struct{} + borrow *memborrow.Ledger + // throttleMemory is set when actor cgroups get memory.high at what the + // actor books, with watchMemory killing the ones stuck there. + throttleMemory bool // actorLogger forwards the actor container's stdout/stderr to the worker pod's // stdout as ate.dev/*-labeled JSON and emits actor lifecycle events (parity @@ -400,20 +420,23 @@ type AteWorkerService struct { var _ ateworkerpb.WorkerServer = (*AteWorkerService)(nil) // NewService creates a new AteWorkerService. -func NewService(podUID, chBinary string, guestDebug bool, memReserveMiB, maxActors int, tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger) *AteWorkerService { - return &AteWorkerService{ - locks: actorlock.New(), - inFlight: actorlock.NewInFlight(), - actors: map[string]*hostedActor{}, - guestSlots: make(chan struct{}, statsFanOut), - maxActors: maxActors, - podUID: podUID, - chBinary: chBinary, - guestDebug: guestDebug, - memReserveMiB: memReserveMiB, - tunnel: tunnel, - actorLogger: actorLogger, - } +func NewService(podUID, chBinary string, guestDebug bool, maxActors int, tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, bootID string) *AteWorkerService { + s := &AteWorkerService{ + locks: actorlock.New(), + inFlight: actorlock.NewInFlight(), + actors: map[string]*hostedActor{}, + guestSlots: make(chan struct{}, statsFanOut), + maxActors: maxActors, + podUID: podUID, + chBinary: chBinary, + guestDebug: guestDebug, + snapshotWrites: make(chan struct{}, snapshotWriteSlots), + tunnel: tunnel, + actorLogger: actorLogger, + bootID: bootID, + } + s.borrow = &memborrow.Ledger{Unbooked: s.unbookedMemory} + return s } // beginRPC registers before checking the drain flag so shutdown cannot miss it. diff --git a/cmd/ateworker-microvm/memlimit.go b/cmd/ateworker-microvm/memlimit.go new file mode 100644 index 0000000000..c9837e9a8e --- /dev/null +++ b/cmd/ateworker-microvm/memlimit.go @@ -0,0 +1,79 @@ +//go:build linux + +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log/slog" + "time" + + "go.ate.dev/substrate/internal/ateattr" + "go.ate.dev/substrate/internal/ateworker/cgroup" +) + +// leafMemoryHigh is the memory.high for an actor's cgroup, or 0 for none: what +// the actor books on the worker. The guest can't use more than its memory +// limit, so past that it's cloud-hypervisor or virtiofsd growing, which would +// otherwise take the whole worker pod to its limit (see cgroup.MemoryGuard). +func (s *AteWorkerService) leafMemoryHigh(memoryBytes int64) int64 { + if !s.throttleMemory || memoryBytes <= 0 { + return 0 + } + return actorBooking(memoryBytes) +} + +// watchMemory kills actors stuck at their cgroup's memory.high, until ctx is +// done. +func (s *AteWorkerService) watchMemory(ctx context.Context) { + ticker := time.NewTicker(cgroup.MemoryGuardInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + for _, h := range s.hostedActors() { + s.checkMemory(ctx, h) + } + } + } +} + +// checkMemory kills h's VMM and virtiofsd if their cgroup is stuck at its +// memory.high. watchVM then reports the actor crashed. +func (s *AteWorkerService) checkMemory(ctx context.Context, h *hostedActor) { + s.actorsMu.RLock() + ra := h.vm + s.actorsMu.RUnlock() + if ra == nil || ra.stopping.Load() { + return + } + kill, ok := ra.memory.Check(ctx, cgroup.Root, h.attribution.UID) + if !ok { + return + } + attrs := append(ateattr.ActorLogAttrs(h.attribution), + slog.Uint64("memoryLimitBytes", kill.Limit), + slog.Uint64(string(ateattr.StatsMemoryUsageKey), kill.Current)) + slog.LogAttrs(ctx, slog.LevelWarn, "Killing a microVM over its memory limit", attrs...) + go func() { + if err := <-kill.Done; err != nil { + attrs = append(attrs, slog.Any("err", err)) + slog.LogAttrs(ctx, slog.LevelError, "Failed to kill a microVM over its memory limit", attrs...) + } + }() +} diff --git a/cmd/ateworker-microvm/memlimit_test.go b/cmd/ateworker-microvm/memlimit_test.go new file mode 100644 index 0000000000..604bfbf35b --- /dev/null +++ b/cmd/ateworker-microvm/memlimit_test.go @@ -0,0 +1,32 @@ +//go:build linux + +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "testing" + +func TestLeafMemoryHigh(t *testing.T) { + throttled := &AteWorkerService{throttleMemory: true} + if got, want := throttled.leafMemoryHigh(1<<30), actorBooking(1<<30); got != want { + t.Errorf("leafMemoryHigh(1GiB) = %d, want the booking %d", got, want) + } + if got := throttled.leafMemoryHigh(0); got != 0 { + t.Errorf("leafMemoryHigh(0) = %d, want 0", got) + } + if got := (&AteWorkerService{}).leafMemoryHigh(1 << 30); got != 0 { + t.Errorf("leafMemoryHigh without PSI = %d, want 0", got) + } +} diff --git a/cmd/ateworker-microvm/net_linux_test.go b/cmd/ateworker-microvm/net_linux_test.go index a6e156cc00..69e27140a4 100644 --- a/cmd/ateworker-microvm/net_linux_test.go +++ b/cmd/ateworker-microvm/net_linux_test.go @@ -53,7 +53,7 @@ func TestHostActorReplacesSameActor(t *testing.T) { } }) for range 2 { - if _, err := service.hostActor(ctx, resources.ActorAttribution{UID: actorUID}, false); err != nil { + if _, err := service.hostActor(ctx, resources.ActorAttribution{UID: actorUID}, false, 0); err != nil { t.Fatal(err) } } diff --git a/cmd/ateworker-microvm/restore.go b/cmd/ateworker-microvm/restore.go index fae3772575..d6d75f541d 100644 --- a/cmd/ateworker-microvm/restore.go +++ b/cmd/ateworker-microvm/restore.go @@ -175,7 +175,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb // so only it counts from the first reading; the lower fidelities cold-boot // a fresh guest whose CPU counts from zero. resumesGuest := includesMemory(req.GetFidelity()) - if _, err := s.hostActor(ctx, attribution, resumesGuest); err != nil { + if _, err := s.hostActor(ctx, attribution, resumesGuest, p.size.MemoryBytes); err != nil { return nil, err } defer func() { @@ -186,6 +186,9 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb _ = s.unhostActor(cleanupCtx, attribution.UID) } }() + if err := s.borrow.Wait(ctx); err != nil { + return nil, err + } // Restore the durable-dir volumes before anything can observe them: for MEMORY // that means before the share's virtiofsd starts, for the lower fidelities @@ -340,7 +343,7 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo return untarErr } tUpper := time.Now() - leaf, err := cgroup.OpenActorLeaf(actorUID, p.size.MilliCPU) + leaf, err := cgroup.OpenActorLeaf(actorUID, p.size.MilliCPU, s.leafMemoryHigh(p.size.MemoryBytes)) if err != nil { return err } @@ -521,7 +524,7 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo ra := &runningActor{ chProc: chProc, vfsdProc: vfsdProc, - apiSocket: apiSocket, baseID: srcID, restoreSourceDir: restoreDir, + apiSocket: apiSocket, memoryBytes: p.size.MemoryBytes, baseID: srcID, restoreSourceDir: restoreDir, preserveRestoreSource: preserveRestoreDir, snapshotIsSelfContained: memMode == ch.MemRestoreEager, // Signaling an id the agent does not know fails the whole graceful @@ -541,6 +544,7 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo if err := s.tunnel.ActivateIngress(p.attribution(), s.sandboxDialer(p.actorUID)); err != nil { return err } + ra.memory.Resume(uint64(s.leafMemoryHigh(p.size.MemoryBytes))) s.setRunningVM(actorUID, ra) go s.watchVM(p.attribution(), p.actorDirs, ra) diff --git a/cmd/ateworker-microvm/run.go b/cmd/ateworker-microvm/run.go index 1b2fda9fc3..dbaba716d9 100644 --- a/cmd/ateworker-microvm/run.go +++ b/cmd/ateworker-microvm/run.go @@ -42,6 +42,7 @@ import ( "go.ate.dev/substrate/internal/ateworkervalidation" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/internal/sizing" @@ -72,6 +73,10 @@ type runningActor struct { // exit from a crash: shutting the VM down makes cloud-hypervisor exit, and // virtiofsd exits once its only client is gone. stopping atomic.Bool + // memoryBytes is the actor's memory limit, what it books on the worker. + memoryBytes int64 + // memory kills the actor's cgroup once it is stuck at its memory.high. + memory cgroup.MemoryGuard // restoreSourceDir is the snapshot dir this actor was OnDemand-restored from // (CH demand-pages its guest RAM from it). Set when restored via OnDemand. @@ -135,30 +140,20 @@ const ( // boots it as PID 1 (init=), so it is the guest's entire userspace. const kataAgentPath = "/usr/bin/kata-agent" -// vmmMemReserveMiB is the DEFAULT guest RAM held back from the pod's memory limit -// for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the same -// pod cgroup as the guest RAM; without a margin the pod OOMs. Overridable per -// deployment via --vmm-mem-reserve-mib (see AteWorkerService.memReserveMiB). -// -// Measured on the worker pod's cgroup with one 256MiB-guest actor: the VMM stack's own -// cost is ~12MiB (anon 8.1 + kernel 3.7; the rest of cloud-hypervisor's RSS is the guest -// memfd, already accounted as guest RAM), and two virtiofsds are ~3MiB each. What needs -// the rest of the margin is transient: a pause/resume cycle took the cgroup from 94MiB -// to 153MiB, and the 57MiB difference was page cache from writing and reading the -// snapshot. -// -// That transient scales with snapshot size, so no fixed reserve is right for every guest -// size — this one is halved rather than cut to the ~32MiB the steady state would justify. -// The fix that would let it drop that far is keeping snapshot I/O out of the page cache -// (posix_fadvise(DONTNEED) after the checkpoint write and the restore read); until then -// the margin absorbs it, and a deployment running large guests can raise the flag. -const vmmMemReserveMiB = 128 - -// minGuestMemMiB is the floor for guest RAM (the declared limit minus the VMM -// reserve); a declared memory limit that leaves less is rejected at cold boot with a -// clear error instead of being silently honored (see resolveGuestMemMiB), since too -// little RAM makes the guest hang on boot rather than fail cleanly. Keep the admission -// floor on ActorTemplate.spec.resources in sync (it is this value + vmmMemReserveMiB). +// actorOverhead is what an actor costs outside its guest, in the same worker +// pod cgroup, which the control plane books with every actor: cloud-hypervisor +// and virtiofsd, and the host page tables for guest RAM, KVM's EPT and the +// memfd's xarray. +var actorOverhead = &ateletpb.RuntimeOverhead{MemoryBytes: 4 << 20, MemoryBytesPerGib: 8 << 20} + +// memoryReservedBytes is how much of the worker pod's memory limit we keep from +// actors: the worker process itself, and the page cache of snapshot writes in +// flight. +const memoryReservedBytes = 32<<20 + snapshotWriteSlots*snapshotWriteHeadroom + +// minGuestMemMiB is the floor for guest RAM, and so for an actor's memory limit. A +// smaller one is rejected at cold boot (see resolveGuestMemMiB), since too little +// RAM makes the guest hang on boot rather than fail cleanly. // // Measured against the counter demo on a guest booting the agent as PID 1: 32MiB never // reaches Ready, 64MiB boots but idles with 1.1MB free (it only survives because page @@ -283,7 +278,7 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run return nil, err } // Publish attribution before boot so stats can include startup usage. - if _, err := s.hostActor(ctx, attribution, false); err != nil { + if _, err := s.hostActor(ctx, attribution, false, p.size.MemoryBytes); err != nil { return nil, err } defer func() { @@ -294,6 +289,9 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run _ = s.unhostActor(cleanupCtx, attribution.UID) } }() + if err := s.borrow.Wait(ctx); err != nil { + return nil, err + } if err := s.coldBootActorRetrying(ctx, p); err != nil { return nil, err @@ -374,7 +372,7 @@ func (s *AteWorkerService) coldBootActorRetrying(ctx context.Context, p actorBoo slog.String("id", p.actorUID), slog.Int("attempt", attempt), slog.Any("err", err)) // The failed attempt deactivated egress, which retires the listener // bound to it, so the network is rebuilt. The actor keeps its slot. - if _, err := s.hostActor(ctx, p.attribution(), false); err != nil { + if _, err := s.hostActor(ctx, p.attribution(), false, p.size.MemoryBytes); err != nil { return err } } @@ -455,8 +453,8 @@ func (s *AteWorkerService) coldBootActor(ctx context.Context, p actorBootParams) kparams := s.kernelParams() // Size the VM to the actor's declared limits (see internal/sizing), which the - // API requires. vCPUs round up; VM RAM reserves a fixed margin for the VMM + - // virtiofsd, which share the pod cgroup with the guest RAM. NB: a MEMORY + // API requires. vCPUs round up; VM RAM is the memory limit, and what the actor + // costs outside the guest is booked as its overhead (see actorOverhead). NB: a MEMORY // snapshot restore reuses the size baked into the snapshot // (restoreMemoryFidelity), so resizing an existing actor takes effect on its // next cold boot. @@ -464,7 +462,7 @@ func (s *AteWorkerService) coldBootActor(ctx context.Context, p actorBootParams) if vcpus == 0 { return errors.New("actor has no cpu limit to size the micro-VM's vCPUs") } - memMiB, err := resolveGuestMemMiB(p.size.MemoryBytes, s.memReserveMiB) + memMiB, err := resolveGuestMemMiB(p.size.MemoryBytes) if err != nil { return err } @@ -481,10 +479,8 @@ func (s *AteWorkerService) coldBootActor(ctx context.Context, p actorBootParams) // template spec is immutable, so an actor's limits were validated when its // golden was built. if err := checkResourceEnvelope(ctrs, guestEnvelope{ - memMiB: memMiB, - vcpus: vcpus, - declaredBytes: p.size.MemoryBytes, - reserveMiB: s.memReserveMiB, + memMiB: memMiB, + vcpus: vcpus, }); err != nil { return err } @@ -499,7 +495,7 @@ func (s *AteWorkerService) coldBootActor(ctx context.Context, p actorBootParams) // host upper, mounted into the shared dir) + durable-dir and CSI volumes (if any), // and start the ONE virtiofsd that serves them all. CH connects to it at vm.create // and demand-pages for the actor's lifetime, so ateworker owns the process (killed in teardownActor). - leaf, err := cgroup.OpenActorLeaf(actorUID, p.size.MilliCPU) + leaf, err := cgroup.OpenActorLeaf(actorUID, p.size.MilliCPU, s.leafMemoryHigh(p.size.MemoryBytes)) if err != nil { return err } @@ -614,10 +610,11 @@ func (s *AteWorkerService) coldBootActor(ctx context.Context, p actorBootParams) slog.Duration("wakeup_probe", time.Since(tContainers)), slog.Duration("since_boot", time.Since(tBooted))) - ra := &runningActor{chProc: chProc, vfsdProc: vfsdProc, apiSocket: apiSocket, baseID: actorUID, guestAgent: ac, workloadIDs: workloadIDs(ctrs)} + ra := &runningActor{chProc: chProc, vfsdProc: vfsdProc, apiSocket: apiSocket, memoryBytes: p.size.MemoryBytes, baseID: actorUID, guestAgent: ac, workloadIDs: workloadIDs(ctrs)} if err := s.tunnel.ActivateIngress(p.attribution(), s.sandboxDialer(p.actorUID)); err != nil { return err } + ra.memory.Resume(uint64(s.leafMemoryHigh(p.size.MemoryBytes))) s.setRunningVM(actorUID, ra) go s.watchVM(p.attribution(), p.actorDirs, ra) @@ -747,19 +744,16 @@ func (s *AteWorkerService) kernelParams() string { } // resolveGuestMemMiB returns the micro-VM guest RAM (MiB) for an actor's declared -// memory limit: the declared memory minus the VMM reserve. It errors if the limit is -// unset or leaves less than a bootable minimum (see vmmMemReserveMiB, minGuestMemMiB, -// and internal/sizing). -func resolveGuestMemMiB(declaredBytes int64, reserveMiB int) (int, error) { +// memory limit, which is all of it (see actorOverhead). It errors if the limit +// is unset or below minGuestMemMiB. +func resolveGuestMemMiB(declaredBytes int64) (int, error) { if declaredBytes <= 0 { return 0, errors.New("actor has no memory limit to size the micro-VM's RAM") } - declaredMiB := int(declaredBytes / (1024 * 1024)) - m := declaredMiB - reserveMiB + m := int(declaredBytes / (1024 * 1024)) if m < minGuestMemMiB { - return 0, fmt.Errorf("actor memory limit %dMiB is too small for a micro-VM: "+ - "the %dMiB VMM reserve leaves %dMiB, below the %dMiB guest minimum", - declaredMiB, reserveMiB, m, minGuestMemMiB) + return 0, fmt.Errorf("actor memory limit %dMiB is too small for a microVM, the minimum is %dMiB", + m, minGuestMemMiB) } return m, nil } diff --git a/cmd/ateworker-microvm/run_test.go b/cmd/ateworker-microvm/run_test.go index ee6b71d55e..6e7c2bd666 100644 --- a/cmd/ateworker-microvm/run_test.go +++ b/cmd/ateworker-microvm/run_test.go @@ -103,13 +103,10 @@ func TestDialAgentRetryContextCanceled(t *testing.T) { } } -// resolveGuestMemMiB must honor a declared limit (minus the VMM reserve) and error -// when the limit is unset or the reserve leaves too little to boot a guest. +// resolveGuestMemMiB gives the guest the whole declared limit, and errors when +// the limit is unset or too small to boot a guest. func TestResolveGuestMemMiB(t *testing.T) { - const ( - mib = 1024 * 1024 - reserve = 256 // vmmMemReserveMiB - ) + const mib = 1024 * 1024 tests := []struct { name string declaredMiB int64 // 0 => unset @@ -117,15 +114,13 @@ func TestResolveGuestMemMiB(t *testing.T) { wantErr bool }{ {name: "unset", declaredMiB: 0, wantErr: true}, - {name: "declared honored minus reserve", declaredMiB: 1536, wantMiB: 1536 - reserve}, - {name: "just above minimum", declaredMiB: reserve + minGuestMemMiB, wantMiB: minGuestMemMiB}, - {name: "reserve exactly swallows limit", declaredMiB: reserve, wantErr: true}, - {name: "limit below reserve", declaredMiB: 128, wantErr: true}, - {name: "boot-hang band (too little guest RAM)", declaredMiB: 320, wantErr: true}, + {name: "declared honored", declaredMiB: 1536, wantMiB: 1536}, + {name: "minimum", declaredMiB: minGuestMemMiB, wantMiB: minGuestMemMiB}, + {name: "below minimum", declaredMiB: minGuestMemMiB - 1, wantErr: true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got, err := resolveGuestMemMiB(tc.declaredMiB*mib, reserve) + got, err := resolveGuestMemMiB(tc.declaredMiB * mib) if tc.wantErr { if err == nil { t.Fatalf("resolveGuestMemMiB(%dMiB) = %d, nil; want an error", tc.declaredMiB, got) diff --git a/cmd/ateworker-microvm/snapshotborrow.go b/cmd/ateworker-microvm/snapshotborrow.go new file mode 100644 index 0000000000..da5868028a --- /dev/null +++ b/cmd/ateworker-microvm/snapshotborrow.go @@ -0,0 +1,37 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// unbookedMemory is the worker's reported memory capacity less what its +// actors booked, overhead included. +func (s *AteWorkerService) unbookedMemory(podLimit int64) int64 { + unbooked := podLimit - memoryReservedBytes + s.actorsMu.RLock() + defer s.actorsMu.RUnlock() + for _, h := range s.actors { + unbooked -= actorBooking(h.memoryBytes) + } + return unbooked +} + +// actorBooking is what the control plane books for an actor with this memory +// limit: the limit plus actorOverhead. +func actorBooking(memoryBytes int64) int64 { + const gib = 1 << 30 + perGiB := actorOverhead.GetMemoryBytesPerGib() + return memoryBytes + actorOverhead.GetMemoryBytes() + memoryBytes/gib*perGiB + memoryBytes%gib*perGiB/gib +} diff --git a/cmd/ateworker-microvm/snapshotborrow_test.go b/cmd/ateworker-microvm/snapshotborrow_test.go new file mode 100644 index 0000000000..6f0312d7c2 --- /dev/null +++ b/cmd/ateworker-microvm/snapshotborrow_test.go @@ -0,0 +1,32 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "testing" + +func TestUnbookedMemory(t *testing.T) { + const limit = 4 << 30 + s := &AteWorkerService{actors: map[string]*hostedActor{ + "a": {memoryBytes: 1 << 30}, + "b": {memoryBytes: 512 << 20}, + }} + // 4MiB each, plus 8MiB per GiB + want := int64(limit - memoryReservedBytes - (1<<30 + 4<<20 + 8<<20) - (512<<20 + 4<<20 + 4<<20)) + if got := s.unbookedMemory(limit); got != want { + t.Errorf("unbookedMemory() = %d, want %d", got, want) + } +} diff --git a/cmd/ateworker-microvm/snapshotcache.go b/cmd/ateworker-microvm/snapshotcache.go index 91ab2618ae..971f540a33 100644 --- a/cmd/ateworker-microvm/snapshotcache.go +++ b/cmd/ateworker-microvm/snapshotcache.go @@ -49,8 +49,9 @@ func dropStagedMemoryImage(ctx context.Context, restoreDir string) { } // dropActorSnapshotCacheAsync pins the actor's snapshot files in restoreDir -// while the caller holds the actor lock, then evicts their clean pages from the -// host page cache in the background. +// while the caller holds the actor lock, then evicts them from the host page +// cache in the background, once they are on disk. A snapshot taken moments ago +// is still being written back. func dropActorSnapshotCacheAsync(restoreDir string, includeMemory bool) { targets := snapshotcache.FindFiles(restoreDir, func(name string) bool { return isSnapshotDataFile(name, includeMemory) @@ -58,5 +59,5 @@ func dropActorSnapshotCacheAsync(restoreDir string, includeMemory bool) { if len(targets) == 0 { return } - go snapshotcache.EvictWithRetries(targets) + go snapshotcache.SyncAndEvict(targets) } diff --git a/cmd/ateworker-microvm/spec.go b/cmd/ateworker-microvm/spec.go index 21eec6af87..3342d95f8a 100644 --- a/cmd/ateworker-microvm/spec.go +++ b/cmd/ateworker-microvm/spec.go @@ -17,7 +17,6 @@ package main import ( - "fmt" "math" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -27,34 +26,24 @@ import ( ) // guestEnvelope is the ceiling an actor's container limits must fit inside. -// declaredBytes is the actor-level memory limit and reserveMiB the guest RAM -// held back for the VMM; together they explain where memMiB came from, so the -// error can name the field the user can actually raise. // -// memMiB is already net of reserveMiB, while vcpus is not. The asymmetry is -// deliberate: an unreduced memory limit would push the worker pod past its own -// and get it OOM-killed, whereas CPU is compressible, so the shortfall only -// slows the workload down. cloud-hypervisor's vCPU threads, virtiofsd and ateworker -// are host processes drawing on the same worker-pod CPU quota, and the -// scheduler's capacity check sets none of it aside, so a container gets somewhat -// less CPU than it declared and the in-guest quota is throttled by the host -// before it binds. Carving out a CPU reserve is left for a follow-up. +// memMiB is the actor's whole memory limit, and vcpus its CPU limit rounded up. +// cloud-hypervisor's vCPU threads, virtiofsd and ateworker are host processes +// drawing on the same worker-pod CPU quota, and the scheduler's capacity check +// sets none of it aside, so a container gets somewhat less CPU than it declared +// and the in-guest quota is throttled by the host before it binds. Carving out a +// CPU reserve is left for a follow-up. type guestEnvelope struct { - memMiB int - vcpus int - declaredBytes int64 - reserveMiB int + memMiB int + vcpus int } // remedy names the field that raises the memory ceiling. func (e guestEnvelope) remedy() string { - return fmt.Sprintf("raise spec.resources.limits.memory (declared %dMiB, less the %dMiB VMM reserve) or lower the container limits", - e.declaredBytes/(1024*1024), e.reserveMiB) + return "raise spec.resources.limits.memory or lower the container limits" } -// cpuRemedy names the field that raises the vCPU ceiling. Unlike remedy, it -// takes no reserve into account: the VMM reserve applies to guest memory, not -// vCPU count (see guestEnvelope). +// cpuRemedy names the field that raises the vCPU ceiling. func (e guestEnvelope) cpuRemedy() string { return "raise spec.resources.limits.cpu or lower the container limits" } diff --git a/cmd/ateworker-microvm/spec_test.go b/cmd/ateworker-microvm/spec_test.go index 05ec03908d..75efcaa5b0 100644 --- a/cmd/ateworker-microvm/spec_test.go +++ b/cmd/ateworker-microvm/spec_test.go @@ -101,7 +101,7 @@ func TestCheckResourceEnvelope(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - err := checkResourceEnvelope(tc.ctrs, guestEnvelope{memMiB: 2048, vcpus: 1, declaredBytes: (2048 + 256) * 1024 * 1024, reserveMiB: 256}) + err := checkResourceEnvelope(tc.ctrs, guestEnvelope{memMiB: 2048, vcpus: 1}) if tc.wantErr == "" { if err != nil { t.Fatalf("checkResourceEnvelope() = %v, want nil", err) @@ -121,25 +121,22 @@ func TestCheckResourceEnvelope(t *testing.T) { } } -// When the actor declared its own size, the guest ceiling comes from that limit -// minus the VMM reserve, so the error must point at the actor's limit rather -// than at the SandboxConfig the user cannot usefully change. +// The guest ceiling is the actor's own memory limit, so the error must point at +// that rather than at the SandboxConfig the user cannot usefully change. func TestCheckResourceEnvelope_ErrorNamesActorLimitWhenDeclared(t *testing.T) { const mib = 1024 * 1024 ctr := actorContainer{name: "hog", spec: &specs.Spec{Linux: &specs.Linux{ Resources: &specs.LinuxResources{Memory: &specs.LinuxMemory{Limit: ptr.To(int64(2048 * mib))}}, }}} - err := checkResourceEnvelope([]actorContainer{ctr}, guestEnvelope{ - memMiB: 768, vcpus: 1, declaredBytes: 1024 * mib, reserveMiB: 256, - }) + err := checkResourceEnvelope([]actorContainer{ctr}, guestEnvelope{memMiB: 1024, vcpus: 1}) if err == nil { t.Fatal("checkResourceEnvelope() = nil, want an error") } if got := apierror.Code(err); got != codes.InvalidArgument { t.Errorf("code = %v, want InvalidArgument", got) } - for _, want := range []string{"hog", "1024", "256", "spec.resources.limits.memory"} { + for _, want := range []string{"hog", "1024", "spec.resources.limits.memory"} { if !strings.Contains(err.Error(), want) { t.Errorf("error %q does not mention %q", err.Error(), want) } @@ -150,16 +147,13 @@ func TestCheckResourceEnvelope_ErrorNamesActorLimitWhenDeclared(t *testing.T) { // spec.resources.limits.memory, even when the actor also declares a memory // limit: raising memory cannot change the vCPU count. func TestCheckResourceEnvelope_CPUErrorNamesCPULimitEvenWithDeclaredMemory(t *testing.T) { - const mib = 1024 * 1024 period := uint64(kata.DefaultCPUPeriodUS) quota := int64(2000 * kata.DefaultCPUPeriodUS / 1000) // 2000m ctr := actorContainer{name: "hog", spec: &specs.Spec{Linux: &specs.Linux{ Resources: &specs.LinuxResources{CPU: &specs.LinuxCPU{Quota: "a, Period: &period}}, }}} - err := checkResourceEnvelope([]actorContainer{ctr}, guestEnvelope{ - memMiB: 2048, vcpus: 1, declaredBytes: 2048 * mib, reserveMiB: 256, - }) + err := checkResourceEnvelope([]actorContainer{ctr}, guestEnvelope{memMiB: 2048, vcpus: 1}) if err == nil { t.Fatal("checkResourceEnvelope() = nil, want an error") } @@ -174,30 +168,22 @@ func TestCheckResourceEnvelope_CPUErrorNamesCPULimitEvenWithDeclaredMemory(t *te } } -// The ceiling containers are measured against is the post-reserve guest, not the -// declared limit and not the SandboxConfig default. RunWorkload gets there by -// calling resolveGuestMemMiB before checkResourceEnvelope; this pins the -// arithmetic those two steps compose into, which nothing else asserts. -func TestCheckResourceEnvelope_MeasuresAgainstPostReserveGuest(t *testing.T) { +// A container may ask for the actor's whole memory limit, since the guest gets +// all of it. RunWorkload gets there by calling resolveGuestMemMiB before +// checkResourceEnvelope; this pins the arithmetic those two steps compose into. +func TestCheckResourceEnvelope_ContainerMayUseWholeLimit(t *testing.T) { const mib = 1024 * 1024 - const declaredMiB, reserveMiB = 1024, 256 + const declaredMiB = 1024 - guestMiB, err := resolveGuestMemMiB(int64(declaredMiB)*mib, reserveMiB) + guestMiB, err := resolveGuestMemMiB(int64(declaredMiB) * mib) if err != nil { t.Fatalf("resolveGuestMemMiB() = %v", err) } - if guestMiB != declaredMiB-reserveMiB { - t.Fatalf("guest = %dMiB, want %dMiB (declared minus reserve)", guestMiB, declaredMiB-reserveMiB) - } - - // A container asking for the full declared limit does not fit the guest, - // because the reserve is not the container's to spend. - ctr := actorContainer{name: "hog", spec: &specs.Spec{Linux: &specs.Linux{ + ctr := actorContainer{name: "whole", spec: &specs.Spec{Linux: &specs.Linux{ Resources: &specs.LinuxResources{Memory: &specs.LinuxMemory{Limit: ptr.To(int64(declaredMiB) * mib)}}, }}} - env := guestEnvelope{memMiB: guestMiB, vcpus: 1, declaredBytes: int64(declaredMiB) * mib, reserveMiB: reserveMiB} - if err := checkResourceEnvelope([]actorContainer{ctr}, env); err == nil { - t.Error("checkResourceEnvelope() = nil, want an error: the declared limit does not fit once the reserve is held back") + if err := checkResourceEnvelope([]actorContainer{ctr}, guestEnvelope{memMiB: guestMiB, vcpus: 1}); err != nil { + t.Errorf("checkResourceEnvelope() = %v, want nil", err) } } diff --git a/cmd/dataplane/envoy/README.md b/cmd/dataplane/envoy/README.md index 7399619fae..29730d1043 100644 --- a/cmd/dataplane/envoy/README.md +++ b/cmd/dataplane/envoy/README.md @@ -21,9 +21,9 @@ cmd/dataplane/envoy/ During a build from source, `ate-setup` builds the image from this directory via `docker buildx`, pushes it to `$KO_DOCKER_REPO/envoy-dataplane`, and its resolved digest replaces the `${ENVOY_DATAPLANE_IMAGE}` placeholder in the router and egress manifests (`manifests/ate-install/atenet-router.yaml`, `manifests/ate-install/atenet-egress.yaml`). -A pre-built install (`ate-setup deploy --image-repo REPO --image-tag TAG`) builds nothing and pins `REPO/envoy-dataplane:TAG` instead, so `REPO` has to hold it. Releases do not publish it; see [`docs/dev/releasing.md`](../../../docs/dev/releasing.md#release-images). +A pre-built install (`ate-setup deploy --image-repo REPO --image-tag TAG`) builds nothing and pins `REPO/envoy-dataplane:TAG` instead, so `REPO` has to hold it. Releases publish it there with the other images; see [`docs/dev/releasing.md`](../../../docs/dev/releasing.md#release-images). -The image targets the same platforms as the ko images: `KO_DEFAULTPLATFORMS`, else `defaultPlatforms` in `.ko.yaml` (linux/amd64 and linux/arm64), else `linux/amd64`. Building another architecture compiles Rust under QEMU, which must be registered with binfmt (e.g. `docker run --privileged --rm tonistiigi/binfmt --install arm64`). +The image targets the same platforms as the ko images: `KO_DEFAULTPLATFORMS`, else `defaultPlatforms` in `.ko.yaml` (linux/amd64 and linux/arm64), else `linux/amd64`. The builder stage runs on the build machine's platform and cross-compiles the Rust module, so building another architecture needs no QEMU. To build the image locally without pushing it: diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index d0ac8f2528..6439371119 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -1,348 +1 @@ -# `kubectl-ate` - -A Kubernetes-native CLI plugin for managing Substrate Actor and Worker lifecycles. - -## Running the CLI - -There are two ways to run the tool, depending on whether you are developing locally or installing it permanently. - -### 1. Install as a native `kubectl` Plugin -You can use `go install` to compile the tool and place the binary directly into your Go bin directory (which should be in your `$PATH`). Because the source folder is named `kubectl-ate`, Kubernetes will automatically recognize the resulting binary! - -```bash -go install ./cmd/kubectl-ate -``` -You can now run it seamlessly anywhere as a native Kubernetes command: `kubectl ate `. - -### 2. Run directly from source (Development) -If you are testing changes to the codebase, you can bypass compilation and run the CLI directly from the source tree: - -```bash -go run ./cmd/kubectl-ate -``` - -## Connection & Auto Port-Forwarding -By default, `kubectl-ate` will automatically read your `~/.kube/config`, discover the `ate-api-server` pods in your cluster, and establish a temporary background port-forward tunnel to execute gRPC calls securely. - -If you prefer to route traffic directly (e.g., through a LoadBalancer or when running natively inside a cluster pod), simply provide the `--endpoint` flag to bypass the tunnel. - -## Tracing -The CLI supports on-demand tracing using the `--trace` flag. When enabled, the CLI will generate a trace ID and signal to the server that it wants the request to be traced. - -**Prerequisites:** - -1. The Google Cloud project must have the **Cloud Trace API** enabled. You can enable it using: -```bash -gcloud services enable cloudtrace.googleapis.com --project=PROJECT_ID -``` - -2. The GKE cluster must have **Managed OpenTelemetry** enabled. Clusters created by -`setup-gcp create cluster` or `setup-gcp bootstrap` always enable it. For a cluster that does -not have it, enable it with: - -```bash -gcloud beta container clusters update CLUSTER_NAME \ - --project=PROJECT_ID \ - --managed-otel-scope=COLLECTION_AND_INSTRUMENTATION_COMPONENTS \ - --location=LOCATION -``` - -**Local (kind):** - -The kind overlay installed by `hack/install-ate-kind.sh --deploy-ate-system` already provisions an in-cluster OpenTelemetry Collector and Jaeger in the `otel-system` namespace. No additional setup is required. - -Port-forward the Jaeger UI and invoke any command with `--trace`: -```bash -kubectl port-forward -n otel-system svc/jaeger 16686:16686 & -kubectl ate get actor my-counter-1 --trace -# open http://localhost:16686 and search for the most recent trace -``` - -## Global Flags -These flags can be appended to any command: - -| Flag | Short | Description | Default | -|---|---|---|---| -| `--kubeconfig` | | Path to your kubeconfig file | `~/.kube/config` | -| `--context` | | Name of the kubeconfig context to use | current context | -| `--endpoint` | | Manual gRPC endpoint override (e.g., `localhost:8080`) | | -| `--token-file` | | Path to a bearer token for ate-api authentication, or `-` for stdin | Kubernetes ServiceAccount token | -| `--output` | `-o` | Output format (`table`, `json`, `yaml`) | `table` | -| `--trace` | | Enable on-demand tracing for the request | `false` | -| `--verbose` | `-v` | Increase output verbosity; may be repeated or stacked (e.g., `-v`, `-vv`, `-vvvvv`) | `0` | - -Pass `--version` (`kubectl ate --version`) to print the CLI version and exit. - ---- - -## Command Reference & Examples - -### Getting Resources -List and inspect the state of actors and workers across the cluster. - -```bash -# List actors in one atespace; -a is shorthand for --atespace -kubectl ate get actors --atespace -kubectl ate get actors -a - -# List actors across all atespaces -kubectl ate get actors -A - -# Get a specific actor by name and output as raw YAML -kubectl ate get actor --atespace -o yaml - -# List all physical workers and see which actors are assigned to them -kubectl ate get workers - -# Filter workers by Kubernetes namespace, assigned-actor atespace, or -# worker pool labels (same flags as `top workers`) -kubectl ate get workers -n -kubectl ate get workers -a -kubectl ate get workers -l -``` - -> **Note:** `get actors` requires either `--atespace ` / `-a ` (one atespace) or `-A`/`--all-atespaces` (all atespaces) — there is no default atespace. Getting a single actor always requires `--atespace`/`-a`, since an actor is addressed by `(atespace, name)`. `-a` (lower-case) scopes to one atespace; `-A` (upper-case) spans all. - -> **Note:** Actors, workers, and actor templates are not Kubernetes CRDs — they live in the Substrate control plane's PostgreSQL database, not `etcd`. `kubectl get actor`, `kubectl get worker`, and `kubectl get actortemplate` will not return anything; only `kubectl ate get …` queries the control plane (see [Actor Templates](#actor-templates)). `kubectl get workerpool` *does* work, because pools are CRDs. - -#### `kubectl ate get actor` output columns - -| Column | Meaning | -|---|---| -| `ATESPACE` | The atespace the actor belongs to. Part of the actor's identity; folded into the storage key as `actor::`. | -| `NAME` | The actor's name. User-provided for application actors; UUID for the golden actor that each template materialises while building its golden tag. | -| `TEMPLATE` | The `ActorTemplate` the actor references, displayed as `/`. | -| `STATE` | Current lifecycle state (`ACTOR_STATE_RESUMING`, `ACTOR_STATE_RUNNING`, `ACTOR_STATE_SUSPENDING`, `ACTOR_STATE_SUSPENDED`, `ACTOR_STATE_CRASHED`, `ACTOR_STATE_DELETING`, `ACTOR_STATE_REVERTING`). | -| `WORKER POD` | The worker pod (namespace/name) currently hosting the actor. Empty while suspended. | -| `WORKER IP` | The pod IP of that worker. Empty while suspended. | -| `VERSION` | Monotonic integer that increments on every state transition (resume / suspend / checkpoint). Useful for distinguishing snapshots. | -| `AGE` | Time elapsed since the actor was created. | - -#### `kubectl ate get worker` output columns - -| Column | Meaning | -|---|---| -| `NAMESPACE` | The `WorkerPool` namespace. | -| `POOL` | The `WorkerPool` name. | -| `POD` | The worker pod name. | -| `STATUS` | `FREE` (idle, ready to receive an actor) or `ASSIGNED` (currently hosting an actor). | -| `ASSIGNED ACTOR` | If `STATUS=ASSIGNED`, the actor reference `/