From 15b58199cb7346c2724f0b8ecafe14dd0b89860e Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 26 Oct 2026 23:41:49 +0000 Subject: [PATCH 01/53] e2e: lower kind atelet snapshot upload delay and update revert tests (#3182) Lower atelet's snapshot upload delay in kind so e2e suites exercise background snapshot upload instead of leaving asynchronous checkpoints pending for the full test run. Update TestUpdateTemplateLifecycle and RevertResumeActor to stop assuming an asynchronous suspend never uploads in the background. TestUpdateTemplateLifecycle now reverts before its asynchronous suspend. RevertResumeActor accepts either snapshot its revert of the paused actor can keep, depending on whether the upload was reported first. The control plane returns Aborted while it applies atelet's upload report, so the tests retry Aborted on actor RPCs that follow an asynchronous suspend. AI use: AI-assisted, I reviewed and tested it - [x] Tests pass - [x] Docs updated Signed-off-by: msau42 --- internal/e2e/resume.go | 11 +- internal/e2e/retry.go | 52 +++++++++ internal/e2e/suites/demo/demo_test.go | 36 +++--- .../e2e/suites/demo/updatetemplate_test.go | 104 +++++++++--------- .../kind/atelet/kustomization.yaml | 3 + 5 files changed, 135 insertions(+), 71 deletions(-) create mode 100644 internal/e2e/retry.go diff --git a/internal/e2e/resume.go b/internal/e2e/resume.go index dadd1b1ad9..70bdf17ee1 100644 --- a/internal/e2e/resume.go +++ b/internal/e2e/resume.go @@ -33,9 +33,10 @@ const resumeCapacityWait = 90 * time.Second // is saturated. The control plane returns ResourceExhausted when no worker // is free and expects callers to wait (the router's parking resumer retries // the same condition); suites running concurrently against shared pools make -// that a normal state in e2e, not a failure. Any other error, or saturation -// outlasting the wait budget, is returned to the caller. Each retry is -// logged so a pass that had to wait stays visible in the test output. +// that a normal state in e2e, not a failure. It also retries Aborted, as +// RetryAborted does. Any other error, or a retried error outlasting the wait +// budget, is returned to the caller. Each retry is logged so a pass that had +// to wait stays visible in the test output. // // TODO: Add runtime diagnostic output for unscheduled pods/actors. a Pending // pod's PodScheduled condition names resources the scheduler cannot satisfy. @@ -50,10 +51,10 @@ func ResumeActorAwaitCapacity(t *testing.T, ctx context.Context, clients *Client deadline := time.Now().Add(resumeCapacityWait) for { resp, err := clients.SubstrateAPI.ResumeActor(ctx, req) - if status.Code(err) != codes.ResourceExhausted || time.Now().After(deadline) { + if c := status.Code(err); (c != codes.ResourceExhausted && c != codes.Aborted) || time.Now().After(deadline) { return resp, err } - t.Logf("ResumeActor %s/%s: pool saturated (%v); retrying", req.GetActor().GetAtespace(), req.GetActor().GetName(), err) + t.Logf("ResumeActor %s/%s: %v; retrying", req.GetActor().GetAtespace(), req.GetActor().GetName(), err) select { case <-ctx.Done(): return nil, ctx.Err() diff --git a/internal/e2e/retry.go b/internal/e2e/retry.go new file mode 100644 index 0000000000..f3684a0ea1 --- /dev/null +++ b/internal/e2e/retry.go @@ -0,0 +1,52 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// abortedRetryWait bounds how long RetryAborted retries. +const abortedRetryWait = 30 * time.Second + +// RetryAborted calls rpc with req, retrying while it fails with Aborted. The +// control plane returns Aborted while another operation holds the actor's +// lease, such as applying atelet's report of a background snapshot upload. +// Tests call an actor's RPCs through it after an asynchronous suspend, since +// that report can come at any time. Any other result, or Aborted outlasting +// the wait, is returned to the caller. +func RetryAborted[Req, Resp any](t *testing.T, ctx context.Context, rpc func(context.Context, Req, ...grpc.CallOption) (Resp, error), req Req) (Resp, error) { + t.Helper() + deadline := time.Now().Add(abortedRetryWait) + for { + resp, err := rpc(ctx, req) + if status.Code(err) != codes.Aborted || time.Now().After(deadline) { + return resp, err + } + t.Logf("%T: %v; retrying", req, err) + select { + case <-ctx.Done(): + var zero Resp + return zero, ctx.Err() + case <-time.After(time.Second): + } + } +} diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 166b77a33c..25bb180b90 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -684,7 +684,7 @@ func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func( // Suspending the actor // t.Logf("Suspending Actor %q...", actorID) - suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapi.SuspendActorRequest{ + suspended, err := e2e.RetryAborted(t, ctx, clients.SubstrateAPI.SuspendActor, &ateapi.SuspendActorRequest{ Actor: &ateapi.ObjectRef{Atespace: demoAtespace, Name: actorID}, Options: &ateapi.SuspendOptions{Mode: ateapi.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }) @@ -987,7 +987,7 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) - if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapi.SuspendActorRequest{ + if _, err := e2e.RetryAborted(t, ctx, clients.SubstrateAPI.SuspendActor, &ateapi.SuspendActorRequest{ Actor: &ateapi.ObjectRef{Atespace: demoAtespace, Name: actorName}, Options: &ateapi.SuspendOptions{Mode: ateapi.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }); err != nil { @@ -1169,16 +1169,20 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } validateCounterResponse(t, resp, "after revert from RUNNING", 2, 2) - // Revert from async suspend (local snapshot). Resume prefers a local - // snapshot over the external one, so clearing the local snapshot is what - // prevents this async suspend's count=2 checkpoint from restoring on the - // next resume. - if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapi.SuspendActorRequest{Actor: actorRef, Options: &ateapi.SuspendOptions{Mode: ateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}); err != nil { + // Revert from async suspend (local snapshot). The revert drops the local + // snapshot and keeps the latest durable one: the count=1 snapshot, or this + // pause's count=2 snapshot if atelet has already reported uploading it. + paused, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapi.SuspendActorRequest{Actor: actorRef, Options: &ateapi.SuspendOptions{Mode: ateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}) + if err != nil { t.Fatalf("failed to pause Actor: %v", err) } waitForActorPaused(ctx, t, clients, actorName) + pausedURI := localSnapshot(paused.GetActor().GetStatus()).GetDurableSnapshot().GetObject().GetSnapshotUri() + if pausedURI == "" { + t.Fatal("pause wrote no durable snapshot URI on local snapshot") + } - reverted, err = clients.SubstrateAPI.RevertActor(ctx, &ateapi.RevertActorRequest{Actor: actorRef}) + reverted, err = e2e.RetryAborted(t, ctx, clients.SubstrateAPI.RevertActor, &ateapi.RevertActorRequest{Actor: actorRef}) if err != nil { t.Fatalf("RevertActor from paused: %v", err) } @@ -1188,8 +1192,14 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj if hasLocalSnapshot(reverted.GetActor().GetStatus()) { t.Fatal("local snapshot pointer survived revert from paused") } - if got := durableSnapshotURI(reverted.GetActor().GetStatus()); got != snapshotURI { - t.Fatalf("snapshot URI after revert from paused = %q, want %q", got, snapshotURI) + var wantCount int + switch kept := durableSnapshotURI(reverted.GetActor().GetStatus()); kept { + case snapshotURI: + wantCount = 2 + case pausedURI: + wantCount = 3 + default: + t.Fatalf("snapshot URI after revert from paused = %q, want %q or the pause's %q", kept, snapshotURI, pausedURI) } // SUSPENDED is rejected with FailedPrecondition rather than no-oping. @@ -1197,8 +1207,8 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj t.Fatalf("RevertActor from SUSPENDED: got %v, want FailedPrecondition", err) } - // Final resume: if the local checkpoint had been restored, the counter - // would advance from 2 to 3. Expecting 2 proves the external snapshot won. + // Final resume restores the snapshot the revert kept, so this call takes + // the counters to 2 from the count=1 snapshot, or to 3 from the pause's. if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapi.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("failed to resume Actor after paused revert: %v", err) } @@ -1208,7 +1218,7 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj if err != nil { t.Fatalf("failed to call actor after paused revert: %v", err) } - validateCounterResponse(t, resp, "after revert from paused", 2, 2) + validateCounterResponse(t, resp, "after revert from paused", wantCount, wantCount) if _, err := clients.SubstrateAPI.DeleteActor(ctx, &ateapi.DeleteActorRequest{Actor: actorRef}); err != nil { t.Fatalf("failed to delete Actor: %v", err) diff --git a/internal/e2e/suites/demo/updatetemplate_test.go b/internal/e2e/suites/demo/updatetemplate_test.go index 2ba987ccea..a39d404e9f 100644 --- a/internal/e2e/suites/demo/updatetemplate_test.go +++ b/internal/e2e/suites/demo/updatetemplate_test.go @@ -197,58 +197,9 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapi.SnapshotFidelity) { t.Errorf("[after template update] expected %q (template B validating the preserved file), got response: %s", want, resp) } - // Suspend asynchronously under template B while status.snapshots still - // holds the last committed snapshot from template A. Resuming from the - // local snapshot restores the local checkpoint (which was captured under - // template B, since templates can only be updated while SUSPENDED) and - // preserves the in-memory counter when preferredFidelity is MEMORY. - t.Logf("Pausing Actor %q under template B...", actorID) - if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapi.SuspendActorRequest{ - Actor: &ateapi.ObjectRef{Atespace: demoAtespace, Name: actorID}, - Options: &ateapi.SuspendOptions{Mode: ateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, - }); err != nil { - t.Fatalf("failed to pause Actor under template B: %v", err) - } - waitForActorPaused(ctx, t, clients, actorID) - - paused, err := clients.SubstrateAPI.GetActor(ctx, &ateapi.GetActorRequest{ - Actor: &ateapi.ObjectRef{Atespace: demoAtespace, Name: actorID}, - }) - if err != nil { - t.Fatalf("failed to get paused Actor: %v", err) - } - if got, want := durableSnapshot(paused.GetStatus()).GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { - t.Errorf("paused Actor external_snapshot.actor_template_uid = %q, want template A's %q", got, want) - } - - t.Logf("Resuming Actor %q from pause under template B...", actorID) - if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapi.ResumeActorRequest{ - Actor: &ateapi.ObjectRef{Atespace: demoAtespace, Name: actorID}, - }); err != nil { - t.Fatalf("failed to resume Actor from pause under template B: %v", err) - } - waitForActorState(ctx, t, clients, actorID, ateapi.ActorState_ACTOR_STATE_RUNNING) - - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorID}) - if err != nil { - t.Fatalf("failed to call actor after pause/resume under template B: %v", err) - } - // The local snapshot was captured under B, so it restores at its own - // fidelity: MEMORY keeps both counters, ROOTFS keeps the rootfs one, and - // VOLUMES neither. - wantMemAfterPause, wantRootfsAfterPause := 2, 2 - switch fidelity { - case ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: - wantMemAfterPause, wantRootfsAfterPause = 1, 1 - case ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: - wantMemAfterPause = 1 - } - validateCounterResponse(t, resp, "after pause/resume under template B", wantMemAfterPause, 4) - validateRootfsCounter(t, resp, "after pause/resume under template B", wantRootfsAfterPause) - // Revert while running under template B: the actor goes back to SUSPENDED // at the external snapshot it still holds, which is template A's. The spec - // still points at B and two sprints have already run under it, so nothing + // still points at B and a sprint has already run under it, so nothing // about the actor says it is repointed except the snapshot's own record of // what captured it — the next resume has to judge by // external_snapshot.actor_template_uid (A) and restore data-only again. @@ -263,8 +214,6 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapi.SnapshotFidelity) { if got := revertedStatus.GetState(); got != ateapi.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("reverted Actor state = %v, want SUSPENDED", got) } - // Resume prefers a local checkpoint over the external snapshot, so one - // surviving here would hide the repoint the next resume has to detect. if hasLocalSnapshot(revertedStatus) { t.Errorf("reverted Actor local_snapshot present in %v, want cleared", revertedStatus.GetSnapshots()) } @@ -300,11 +249,60 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapi.SnapshotFidelity) { t.Errorf("[after revert under template B] expected %q (template B reading the rewound file), got response: %s", want, resp) } + // Suspend asynchronously under template B while status.snapshots still + // holds the last committed snapshot from template A. Resuming from the + // local snapshot restores the local checkpoint (which was captured under + // template B, since templates can only be updated while SUSPENDED) and + // preserves the in-memory counter when preferredFidelity is MEMORY. + t.Logf("Pausing Actor %q under template B...", actorID) + paused, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapi.SuspendActorRequest{ + Actor: &ateapi.ObjectRef{Atespace: demoAtespace, Name: actorID}, + Options: &ateapi.SuspendOptions{Mode: ateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }) + if err != nil { + t.Fatalf("failed to pause Actor under template B: %v", err) + } + pausedLocal := localSnapshot(paused.GetActor().GetStatus()) + if pausedLocal == nil { + t.Fatalf("paused Actor holds no local snapshot: %v", paused.GetActor().GetStatus()) + } + if got, want := pausedLocal.GetActorTemplateUid(), createdB.GetMetadata().GetUid(); got != want { + t.Errorf("paused Actor local snapshot actor_template_uid = %q, want template B's %q", got, want) + } + if got, want := durableSnapshot(paused.GetActor().GetStatus()).GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { + t.Errorf("paused Actor durable snapshot actor_template_uid = %q, want template A's %q", got, want) + } + + t.Logf("Resuming Actor %q from pause under template B...", actorID) + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapi.ResumeActorRequest{ + Actor: &ateapi.ObjectRef{Atespace: demoAtespace, Name: actorID}, + }); err != nil { + t.Fatalf("failed to resume Actor from pause under template B: %v", err) + } + waitForActorState(ctx, t, clients, actorID, ateapi.ActorState_ACTOR_STATE_RUNNING) + + resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorID}) + if err != nil { + t.Fatalf("failed to call actor after pause/resume under template B: %v", err) + } + // The local snapshot was captured under B, so it restores at its own + // fidelity: MEMORY keeps both counters, ROOTFS keeps the rootfs one, and + // VOLUMES neither. + wantMemAfterPause, wantRootfsAfterPause := 2, 2 + switch fidelity { + case ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + wantMemAfterPause, wantRootfsAfterPause = 1, 1 + case ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + wantMemAfterPause = 1 + } + validateCounterResponse(t, resp, "after pause/resume under template B", wantMemAfterPause, 4) + validateRootfsCounter(t, resp, "after pause/resume under template B", wantRootfsAfterPause) + // A second suspend closes the loop: the new snapshot is captured under B, // so the actor is no longer repointed and the next resume restores at the // snapshot's own scope rather than being forced down to data-only. t.Logf("Suspending Actor %q again...", actorID) - if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapi.SuspendActorRequest{ + if _, err := e2e.RetryAborted(t, ctx, clients.SubstrateAPI.SuspendActor, &ateapi.SuspendActorRequest{ Actor: &ateapi.ObjectRef{Atespace: demoAtespace, Name: actorID}, Options: &ateapi.SuspendOptions{Mode: ateapi.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, }); err != nil { diff --git a/manifests/ate-install/kind/atelet/kustomization.yaml b/manifests/ate-install/kind/atelet/kustomization.yaml index 0616973c6b..b18e13a69e 100644 --- a/manifests/ate-install/kind/atelet/kustomization.yaml +++ b/manifests/ate-install/kind/atelet/kustomization.yaml @@ -37,6 +37,9 @@ patches: # Kind clusters are dev/CI: run atelet at debug so e2e suites can # assert on per-item log lines. Production installs default to info. - --log-level=debug + # Upload asynchronous snapshots in the background 2s after a + # suspend (5m by default), so e2e suites run alongside uploads. + - --snapshot-upload-delay=2s # The OTLP endpoint comes from the ate-otel-config ConfigMap the # atelet DaemonSet consumes via envFrom; the kind copy lives at # ../ate-otel-config.yaml and is applied by the parent overlay or From e0350cd9532fd89f18a5a1cfed8ffd30b8d57b5d Mon Sep 17 00:00:00 2001 From: shrutiyam-glitch Date: Tue, 27 Oct 2026 00:08:04 +0000 Subject: [PATCH 02/53] atelet: emit a Warning Event on worker pods when syncing sandbox runtimes fails (#3146) Fixes #3145 When `workerSandboxSyncer` in `cmd/atelet` fails to build or write a worker pod's `sandbox-runtimes.json`, `atelet` only logs a warning on the node. `atecontroller` emits events on the `WorkerPool` when a `SandboxConfig` reference does not resolve, and `ateapi` only hears from `ateworker` after `sandbox-runtimes.json` changes, so node-side failures (unreachable asset URLs, SHA256 mismatches, missing architecture assets, or failed `runsc cpu-features` probes) are only visible in `atelet` container logs. Pass an `events.k8s.io/v1` `EventRecorder` into `workerSandboxSyncer` and emit a `Warning` `SandboxRuntimeSyncFailed` (`action: SyncSandboxRuntimes`) `Event` on the worker `Pod` in `finish` for `errPermanent`, `errIncomplete`, non-`errUnresolved` retries, and `errUnresolved` when giving up at `syncMaxRetries`, attaching the resolved `SandboxConfig` (or `WorkerPool` when its config has not resolved) as `related` so config edits start a new event series with the updated error message. Truncate event notes at the 1024-byte `events.k8s.io/v1` limit on a UTF-8 rune boundary, and grant the `atelet` `ClusterRole` `create` and `patch` on `events.k8s.io` `events`. AI-assisted, I reviewed and tested it. - [x] Tests pass - [x] Docs updated --------- Signed-off-by: Shrutiya --- cmd/atelet/main.go | 23 +++++- cmd/atelet/runtime_prewarm_test.go | 9 ++- cmd/atelet/worker_sandbox_syncer.go | 76 +++++++++++++++--- cmd/atelet/worker_sandbox_syncer_test.go | 93 ++++++++++++++++++++-- cmd/atelet/worker_sandbox_versions.go | 42 +++++++--- cmd/atelet/worker_sandbox_versions_test.go | 14 ++-- docs/api-guide.md | 2 +- manifests/ate-install/atelet.yaml | 5 ++ 8 files changed, 224 insertions(+), 40 deletions(-) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index aac1126071..2f708a8778 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -85,10 +85,14 @@ import ( "google.golang.org/grpc/status" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/events" "k8s.io/utils/clock" "k8s.io/utils/lru" ) @@ -479,6 +483,12 @@ func main() { // still covers correctness. prewarmer := newRuntimePrewarmer(runtimes, wmService) go prewarmer.run(ctx) + eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: k8sClient.EventsV1()}) + if err := eventBroadcaster.StartRecordingToSinkWithContext(ctx); err != nil { + serverboot.Fatal(ctx, "Failed to start the event broadcaster", err) + } + defer eventBroadcaster.Shutdown() + // Keep a sandbox runtimes file in each local worker's directory: the // runtimes of the enabled versions of its pool's SandboxConfig, // rewritten when the config changes. @@ -487,7 +497,8 @@ func main() { ateFactory.Ate().V1().WorkerPools(), ateFactory.Ate().V1().SandboxConfigs(), wmService, - nodepath.WorkersDir(), runtimes, prewarmer) + nodepath.WorkersDir(), runtimes, prewarmer, + eventBroadcaster.NewRecorder(newEventScheme(), "atelet")) if err != nil { // Without the syncer no runtime is ever indexed, so no actor could run. serverboot.Fatal(ctx, "Failed to start the worker sandbox runtimes syncer", err) @@ -554,6 +565,16 @@ func main() { slog.InfoContext(ctx, "Shutdown complete") } +// newEventScheme returns a Scheme with core Kubernetes and ATE CRD types +// registered so the event recorder can resolve ObjectReferences for both +// regarding (Pod) and related (SandboxConfig, WorkerPool). +func newEventScheme() *k8sruntime.Scheme { + s := k8sruntime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(s)) + utilruntime.Must(atev1.AddToScheme(s)) + return s +} + // servingCheck is the readiness check that holds /readyz at 503 until served // is closed, which main does once the gRPC port is open. An atelet that // stops before then never closes it, and so is never ready. diff --git a/cmd/atelet/runtime_prewarm_test.go b/cmd/atelet/runtime_prewarm_test.go index 811c97e817..8c8d1778bb 100644 --- a/cmd/atelet/runtime_prewarm_test.go +++ b/cmd/atelet/runtime_prewarm_test.go @@ -34,6 +34,7 @@ import ( k8sruntime "k8s.io/apimachinery/pkg/runtime" kubeinformers "k8s.io/client-go/informers" kubefake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/events" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" @@ -316,7 +317,7 @@ func TestSyncPodPrewarmsDefaultRuntime(t *testing.T) { if err := os.MkdirAll(stale, 0o700); err != nil { t.Fatal(err) } - if err := s.syncPod(context.Background(), pod); err != nil { + if _, err := s.syncPod(context.Background(), pod); err != nil { t.Fatalf("syncPod: %v", err) } wr, ok := s.index.worker("pod-a") @@ -344,7 +345,7 @@ func TestSyncPodPrewarmsDefaultRuntime(t *testing.T) { if err := os.WriteFile(marker, nil, 0o600); err != nil { t.Fatal(err) } - if err := s.syncPod(context.Background(), pod); err != nil { + if _, err := s.syncPod(context.Background(), pod); err != nil { t.Fatalf("syncPod again: %v", err) } if got := rec.items(); len(got) != 2 || got[1] != want[0] { @@ -370,7 +371,7 @@ func TestSyncPodPrewarmsDefaultRuntime(t *testing.T) { if err := os.MkdirAll(previous, 0o700); err != nil { t.Fatal(err) } - if err := s.syncPod(context.Background(), pod); err == nil { + if _, err := s.syncPod(context.Background(), pod); err == nil { t.Fatal("syncPod with the default's probe failing = nil, want an error") } if got := rec.items(); len(got) != 0 { @@ -459,7 +460,7 @@ func TestDefaultRuntimePrewarmThroughSyncer(t *testing.T) { ateFactory.Ate().V1().WorkerPools(), ateFactory.Ate().V1().SandboxConfigs(), newScriptAssetFetcher(t), // the syncer's runsc probe; prewarm goes through herder - workersDir, index, p) + workersDir, index, p, &events.FakeRecorder{}) if err != nil { t.Fatalf("startWorkerSandboxSyncer: %v", err) } diff --git a/cmd/atelet/worker_sandbox_syncer.go b/cmd/atelet/worker_sandbox_syncer.go index 45a82a7488..68ae5edc50 100644 --- a/cmd/atelet/worker_sandbox_syncer.go +++ b/cmd/atelet/worker_sandbox_syncer.go @@ -22,7 +22,8 @@ package main // - process: for each UID, sync the pod, or remove its worker directory // once the pod is gone // - finish: forget the UID, or retry it with backoff, capped at -// syncMaxRetries while the pool's SandboxConfig is unresolved +// syncMaxRetries while the pool's SandboxConfig is unresolved, and +// emit a Warning Event on the pod when a sync fails // // worker_sandbox_versions.go builds the sandbox runtimes file for each // worker pod. @@ -39,12 +40,15 @@ import ( "strings" "sync" "time" + "unicode/utf8" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" + k8sruntime "k8s.io/apimachinery/pkg/runtime" coreinformers "k8s.io/client-go/informers/core/v1" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/events" "k8s.io/client-go/util/workqueue" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" @@ -92,6 +96,18 @@ const syncMaxRetries = 8 // cached syncs for other pods on the node. const syncWorkers = 2 +const ( + // eventReasonSandboxRuntimeSyncFailed is the reason of the Warning + // event emitted on a worker pod when its sandbox runtimes file cannot + // be synced (or is written without a restorable version). + eventReasonSandboxRuntimeSyncFailed = "SandboxRuntimeSyncFailed" + // eventActionSyncSandboxRuntimes is the action of events emitted while + // syncing a worker pod's sandbox runtimes file. + eventActionSyncSandboxRuntimes = "SyncSandboxRuntimes" + // maxEventNoteBytes is the events.k8s.io/v1 Note length limit (1 kB). + maxEventNoteBytes = 1024 +) + // workerSandboxSyncer keeps a sandbox runtimes file in each local worker's // directory under workersDir: the runtimes of the enabled versions of the // SandboxConfig the worker's pool uses. It creates a worker's directory once @@ -106,6 +122,9 @@ type workerSandboxSyncer struct { // so they are fetched before an actor asks for them and stay cached // while none runs them. prewarm runtimePrewarm + // recorder emits a Warning event on the worker pod when syncing its + // sandbox runtimes file fails. + recorder events.EventRecorder // pods holds this node's worker pods, indexed by podUIDIndex. pods cache.Indexer assets sandboxAssetFetcher @@ -139,7 +158,7 @@ type workerSandboxSyncer struct { microvmAssets map[string]bool } -func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, assets sandboxAssetFetcher, workersDir string, index *runtimeIndex, prewarm runtimePrewarm) *workerSandboxSyncer { +func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, assets sandboxAssetFetcher, workersDir string, index *runtimeIndex, prewarm runtimePrewarm, recorder events.EventRecorder) *workerSandboxSyncer { return &workerSandboxSyncer{ resolver: resolver, pods: pods, @@ -147,6 +166,7 @@ func newWorkerSandboxSyncer(resolver *workerSandboxResolver, pods cache.Indexer, scratchDir: ateletpath.SandboxAssetScratchDir, index: index, prewarm: prewarm, + recorder: recorder, // Backoff doubles from 1s for informer lag and unresolved pools, and // caps at 5m where uncapped retries for transient asset fetch or probe // failures stay until they recover. @@ -229,7 +249,7 @@ func (s *workerSandboxSyncer) syncDefaultRuntimeDirs(uid string, defaults []stri // or a list/watch fails (e.g. Forbidden while an RBAC rollout lags the // binary), existing files and index entries stay as they are and only // changes wait. -func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInformer, workerPools ateinformers.WorkerPoolInformer, configs ateinformers.SandboxConfigInformer, assets sandboxAssetFetcher, workersDir string, index *runtimeIndex, prewarm runtimePrewarm) (*workerSandboxSyncer, error) { +func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInformer, workerPools ateinformers.WorkerPoolInformer, configs ateinformers.SandboxConfigInformer, assets sandboxAssetFetcher, workersDir string, index *runtimeIndex, prewarm runtimePrewarm, recorder events.EventRecorder) (*workerSandboxSyncer, error) { if err := pods.Informer().AddIndexers(cache.Indexers{podUIDIndex: func(obj any) ([]string, error) { pod, ok := obj.(*corev1.Pod) if !ok { @@ -243,7 +263,7 @@ func startWorkerSandboxSyncer(ctx context.Context, pods coreinformers.PodInforme workerPools: workerPools.Lister(), configs: configs.Lister(), } - s := newWorkerSandboxSyncer(resolver, pods.Informer().GetIndexer(), assets, workersDir, index, prewarm) + s := newWorkerSandboxSyncer(resolver, pods.Informer().GetIndexer(), assets, workersDir, index, prewarm, recorder) if err := watch(ctx, "worker pod", pods.Informer(), s.podHandler()); err != nil { return nil, err } @@ -444,6 +464,7 @@ func (s *workerSandboxSyncer) run(ctx context.Context, synced ...cache.InformerS // it is gone. func (s *workerSandboxSyncer) process(ctx context.Context, uid string) { defer s.queue.Done(uid) + var related k8sruntime.Object pod, err := s.podByUID(uid) switch { case err != nil: @@ -452,9 +473,9 @@ func (s *workerSandboxSyncer) process(ctx context.Context, uid string) { case pod.DeletionTimestamp != nil: // The Delete event removes the directory. default: - err = s.syncPod(ctx, pod) + related, err = s.syncPod(ctx, pod) } - s.finish(ctx, uid, err) + s.finish(ctx, pod, related, uid, err) } // podByUID returns the cached pod with the given UID, or nil if there is none. @@ -475,32 +496,69 @@ func (s *workerSandboxSyncer) podByUID(uid string) (*corev1.Pod, error) { } // finish forgets uid on success or errPermanent, caps errUnresolved retries at -// syncMaxRetries, and otherwise requeues with backoff without a retry cap. A -// sync that fails before it writes leaves the existing file in place. -func (s *workerSandboxSyncer) finish(ctx context.Context, uid string, err error) { +// syncMaxRetries, and otherwise requeues with backoff without a retry cap. It +// emits a Warning Event on pod for every failure except an unresolved pool +// still being retried. A sync that fails before it writes leaves the existing +// file in place. +func (s *workerSandboxSyncer) finish(ctx context.Context, pod *corev1.Pod, related k8sruntime.Object, uid string, err error) { switch { case err == nil: s.queue.Forget(uid) case errors.Is(err, errPermanent): slog.WarnContext(ctx, "Cannot sync worker sandbox runtimes; not retrying until the pool's SandboxConfig changes or atelet restarts", slog.String("pod_uid", uid), slog.Any("err", err)) + s.recordSyncFailure(pod, related, err) s.queue.Forget(uid) case ctx.Err() != nil: // Shutting down, not a sync failure. case errors.Is(err, errIncomplete): slog.WarnContext(ctx, "Worker sandbox runtimes file is missing a restorable version; will retry", slog.String("pod_uid", uid), slog.Int("retries", s.queue.NumRequeues(uid)), slog.Any("err", err)) + s.recordSyncFailure(pod, related, err) s.queue.AddRateLimited(uid) case s.queue.NumRequeues(uid) < syncMaxRetries || !errors.Is(err, errUnresolved): slog.WarnContext(ctx, "Could not sync worker sandbox runtimes; will retry", slog.String("pod_uid", uid), slog.Int("retries", s.queue.NumRequeues(uid)), slog.Any("err", err)) + if !errors.Is(err, errUnresolved) { + s.recordSyncFailure(pod, related, err) + } s.queue.AddRateLimited(uid) default: slog.WarnContext(ctx, "Could not sync worker sandbox runtimes; giving up until the pool's SandboxConfig changes or atelet restarts", slog.String("pod_uid", uid), slog.Int("retries", syncMaxRetries), slog.Any("err", err)) + s.recordSyncFailure(pod, related, err) s.queue.Forget(uid) } } +// recordSyncFailure emits a Warning event on pod when a sync fails. The +// broadcaster keys its series cache by (regarding, related, reason, action), +// including both objects' ResourceVersion, so retries against the same pod and +// SandboxConfig (or WorkerPool) revisions update the series count instead of +// creating a new Event on each attempt, while a pod status update, config edit, +// or config switch starts a new Event. A message that changes while both +// ResourceVersions stay the same still joins the open series and keeps the +// first attempt's Note. +func (s *workerSandboxSyncer) recordSyncFailure(pod *corev1.Pod, related k8sruntime.Object, err error) { + if pod == nil { + return + } + s.recorder.Eventf(pod, related, corev1.EventTypeWarning, eventReasonSandboxRuntimeSyncFailed, eventActionSyncSandboxRuntimes, "%s", truncateEventNote(err.Error())) +} + +// truncateEventNote caps msg at maxEventNoteBytes on a UTF-8 rune boundary, +// ending a truncated message with "...". +func truncateEventNote(msg string) string { + if len(msg) <= maxEventNoteBytes { + return msg + } + const ellipsis = "..." + cut := maxEventNoteBytes - len(ellipsis) + for cut > 0 && !utf8.RuneStart(msg[cut]) { + cut-- + } + return msg[:cut] + ellipsis +} + func (s *workerSandboxSyncer) removeDir(ctx context.Context, uid string) error { s.index.forget(uid) if err := os.RemoveAll(s.podDir(uid)); err != nil { diff --git a/cmd/atelet/worker_sandbox_syncer_test.go b/cmd/atelet/worker_sandbox_syncer_test.go index 0f0d1b108e..b82b07bdbc 100644 --- a/cmd/atelet/worker_sandbox_syncer_test.go +++ b/cmd/atelet/worker_sandbox_syncer_test.go @@ -25,6 +25,7 @@ import ( "testing" "testing/synctest" "time" + "unicode/utf8" "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" @@ -34,6 +35,8 @@ import ( kubeinformers "k8s.io/client-go/informers" kubefake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/events" + "k8s.io/client-go/tools/reference" "k8s.io/client-go/util/workqueue" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" @@ -171,7 +174,7 @@ func newTestSyncer(t *testing.T, objs ...any) *workerSandboxSyncer { s := newWorkerSandboxSyncer(&workerSandboxResolver{ workerPools: atelisters.NewWorkerPoolLister(pools), configs: atelisters.NewSandboxConfigLister(configs), - }, pods, newTestAssetFetcher(), t.TempDir(), newRuntimeIndex(), nil) + }, pods, newTestAssetFetcher(), t.TempDir(), newRuntimeIndex(), nil, events.NewFakeRecorder(64)) // The queue starts a goroutine; stop it with the test. t.Cleanup(s.queue.ShutDown) s.scratchDir = t.TempDir() // buildFile stages assets there @@ -389,7 +392,7 @@ func TestWorkerSandboxSyncerRegistersAheadOfJitter(t *testing.T) { s.sandboxConfigHandler().OnUpdate(multiVersionConfig(), cfg) synctest.Wait() - plan, err := s.resolver.plan(pod) + plan, _, err := s.resolver.plan(pod) if err != nil { t.Fatal(err) } @@ -414,6 +417,18 @@ func TestWorkerSandboxSyncerRegistersAheadOfJitter(t *testing.T) { }) } +// testEventRecorder wraps FakeRecorder to capture the related object passed to +// Eventf alongside the formatted event string. +type testEventRecorder struct { + *events.FakeRecorder + related k8sruntime.Object +} + +func (r *testEventRecorder) Eventf(regarding, related k8sruntime.Object, eventtype, reason, action, note string, args ...any) { + r.related = related + r.FakeRecorder.Eventf(regarding, related, eventtype, reason, action, note, args...) +} + func TestWorkerSandboxSyncerProcess(t *testing.T) { ctx := context.Background() deleting := workerPod("pod-a", "pool") @@ -439,6 +454,11 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { // indexed registers pod-a in the index before processing, as an // earlier sync would have; wantIndexed is whether it is there after. indexed, wantIndexed bool + // wantEvent is a substring of the Warning event emitted on pod-a, or "" + // when no event should be emitted. wantRelated is the secondary object + // attached to that event. + wantEvent string + wantRelated k8sruntime.Object }{{ name: "writes the file", wantIndexed: true, @@ -498,6 +518,13 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { name: "unresolved pool gives up after syncMaxRetries", objs: []any{workerPod("pod-a", "pool"), multiVersionConfig()}, priorRetries: syncMaxRetries, + wantEvent: "Warning SandboxRuntimeSyncFailed", + }, { + name: "unresolved pool config gives up after syncMaxRetries", + objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "")}, + priorRetries: syncMaxRetries, + wantEvent: "Warning SandboxRuntimeSyncFailed", + wantRelated: resolvedPool("pool", ""), }, { // A transient failure on the default version leaves the existing file // in place and requeues without the syncMaxRetries cap, since no @@ -515,6 +542,8 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { wantDir: true, wantUntouched: true, wantRetries: syncMaxRetries + 1, + wantEvent: "temporary download failure on default version", + wantRelated: multiVersionConfig(), }, { // A transient failure on a restorable version writes the default // runtime right away and requeues without the syncMaxRetries cap. @@ -533,13 +562,36 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { want: gvisorFile(wantRuntime("v2")), wantDir: true, wantRetries: syncMaxRetries + 1, + wantEvent: "temporary download failure on restorable version", + wantRelated: multiVersionConfig(), + }, { + name: "pod of no pool is not retried", + objs: []any{workerPod("pod-a", ""), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + wantEvent: "Warning SandboxRuntimeSyncFailed", }, { - name: "pod of no pool is not retried", - objs: []any{workerPod("pod-a", ""), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + name: "long sync error is truncated on a UTF-8 rune boundary to the Event note limit", + objs: []any{workerPod("pod-a", "pool"), resolvedPool("pool", "gvisor-release"), multiVersionConfig()}, + existing: stale, + assets: &fakeAssetFetcher{ + fn: func(context.Context, *sandboxAssetsRecord) (map[string]string, error) { + // Prefix "SandboxConfig \"gvisor-release\", version \"v2\": " is 46 + // bytes, so 2-byte "é" runes start at even offsets and the + // 1021-byte cut point lands on a continuation byte. + return nil, errors.New(strings.Repeat("é", 1000)) + }, + }, + want: stale, + wantDir: true, + wantUntouched: true, + wantRetries: 1, + wantEvent: strings.Repeat("é", 487) + "...", + wantRelated: multiVersionConfig(), }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := newTestSyncer(t, tt.objs...) + recorder := &testEventRecorder{FakeRecorder: s.recorder.(*events.FakeRecorder)} + s.recorder = recorder if tt.assets != nil { s.assets = tt.assets } @@ -575,6 +627,37 @@ func TestWorkerSandboxSyncerProcess(t *testing.T) { s.process(ctx, uid) + var gotEvent string + select { + case gotEvent = <-recorder.Events: + default: + } + if tt.wantEvent == "" && gotEvent != "" { + t.Errorf("unexpected event %q", gotEvent) + } + if tt.wantEvent != "" && !strings.Contains(gotEvent, tt.wantEvent) { + t.Errorf("event = %q, want substring %q", gotEvent, tt.wantEvent) + } + if gotEvent != "" { + note := strings.TrimPrefix(gotEvent, "Warning SandboxRuntimeSyncFailed ") + if len(note) > maxEventNoteBytes || !utf8.ValidString(note) { + t.Errorf("event note len = %d (max %d), valid UTF-8 = %v", len(note), maxEventNoteBytes, utf8.ValidString(note)) + } + } + if diff := cmp.Diff(tt.wantRelated, recorder.related); diff != "" { + t.Errorf("event related mismatch (-want +got):\n%s", diff) + } + if recorder.related != nil { + if _, err := reference.GetReference(newEventScheme(), recorder.related); err != nil { + t.Errorf("reference.GetReference(%T) error = %v", recorder.related, err) + } + } + select { + case extra := <-recorder.Events: + t.Errorf("unexpected second event %q", extra) + default: + } + if _, err := os.Stat(dir); (err == nil) != tt.wantDir { t.Errorf("directory exists = %v (%v), want %v", err == nil, err, tt.wantDir) } @@ -851,7 +934,7 @@ func TestStartWorkerSandboxSyncer(t *testing.T) { ateFactory.Ate().V1().WorkerPools(), ateFactory.Ate().V1().SandboxConfigs(), newScriptAssetFetcher(t), - dir, newRuntimeIndex(), nil) + dir, newRuntimeIndex(), nil, &events.FakeRecorder{}) if err != nil { t.Fatalf("startWorkerSandboxSyncer() error = %v", err) } diff --git a/cmd/atelet/worker_sandbox_versions.go b/cmd/atelet/worker_sandbox_versions.go index 6449dcbba0..646286507d 100644 --- a/cmd/atelet/worker_sandbox_versions.go +++ b/cmd/atelet/worker_sandbox_versions.go @@ -43,6 +43,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" "go.ate.dev/substrate/internal/hardware" @@ -136,13 +137,20 @@ func (r *workerSandboxResolver) config(pool *atev1.WorkerPool) (*atev1.SandboxCo return cfg, nil } -// plan resolves pod's SandboxConfig and the versions of it this node can run. -func (r *workerSandboxResolver) plan(pod *corev1.Pod) (*sandboxPlan, error) { +// plan resolves pod's SandboxConfig and the versions of it this node can run, +// along with the object the sync is acting on (the resolved SandboxConfig, or +// the WorkerPool when its config has not resolved) for failure Events. +func (r *workerSandboxResolver) plan(pod *corev1.Pod) (*sandboxPlan, k8sruntime.Object, error) { pool, err := r.pool(pod) if err != nil { - return nil, err + return nil, nil, err } - return r.planPool(pool) + cfg, err := r.config(pool) + if err != nil { + return nil, pool, err + } + plan, err := planConfig(cfg) + return plan, cfg, err } // planPool is plan for a pool already in hand: the config its workers run @@ -152,6 +160,12 @@ func (r *workerSandboxResolver) planPool(pool *atev1.WorkerPool) (*sandboxPlan, if err != nil { return nil, err } + return planConfig(cfg) +} + +// planConfig returns the plan for cfg: its class and the versions of it this +// node can run. +func planConfig(cfg *atev1.SandboxConfig) (*sandboxPlan, error) { vers, err := planVersions(cfg) if err != nil { return nil, err @@ -486,30 +500,32 @@ func bundleHash(rec *sandboxAssetsRecord) string { // // A file missing restorable runtimes for a transient reason is written all // the same, and the errIncomplete from build is returned so finish retries. -func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) error { - plan, err := s.resolver.plan(pod) +// The returned k8sruntime.Object is the resolved SandboxConfig (or WorkerPool +// when its config has not resolved), attached as `related` on failure Events. +func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) (k8sruntime.Object, error) { + plan, related, err := s.resolver.plan(pod) if err != nil { - return err + return related, err } hashed := hashVersions(plan) // Resolvable, and on disk, before the file can let the worker register // any of them. if err := s.index.registerRuntimes(plan.config, hashed); err != nil { - return err + return related, err } want, buildErr := s.buildFile(ctx, plan, hashed) if buildErr != nil && !errors.Is(buildErr, errIncomplete) { - return buildErr + return related, buildErr } if err := s.write(ctx, string(pod.UID), plan.config, want); err != nil { - return err + return related, err } // What the worker advertises is what its file now says: a sync that // failed before the write left the previous entry, and one that wrote // a partial file names only the runtimes in it. One class per pool // today; a pool that runs several will yield one plan per class here. if err := s.index.register(string(pod.UID), advertisedClass{class: plan.sandboxClass, config: plan.config, hashed: writtenVersions(hashed, want)}); err != nil { - return err + return related, err } // Only now, with the file naming its defaults: a sync that failed above // left the file, and so the directories, as they were. The build staged @@ -524,13 +540,13 @@ func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) erro } } if err := s.syncDefaultRuntimeDirs(string(pod.UID), defaults); err != nil { - return err + return related, err } for _, hash := range defaults { s.prewarm.enqueue(hash, s.defaultRuntimeDir(string(pod.UID), hash)) } } - return buildErr + return related, buildErr } // writtenVersions returns the versions of hashed whose runtime file names, diff --git a/cmd/atelet/worker_sandbox_versions_test.go b/cmd/atelet/worker_sandbox_versions_test.go index 827ad39c17..100a9ae3d7 100644 --- a/cmd/atelet/worker_sandbox_versions_test.go +++ b/cmd/atelet/worker_sandbox_versions_test.go @@ -266,7 +266,7 @@ func wantVersions() *sandboxruntimes.File { // planAndBuild runs the plan and build steps of syncPod for pod. func planAndBuild(s *workerSandboxSyncer, pod *corev1.Pod) (*sandboxruntimes.File, error) { - plan, err := s.resolver.plan(pod) + plan, _, err := s.resolver.plan(pod) if err != nil { return nil, err } @@ -541,7 +541,7 @@ func TestSyncPodMicroVMAssetCacheAndEviction(t *testing.T) { recPrewarm := &recordingPrewarm{} s.prewarm = recPrewarm - if err := s.syncPod(ctx, workerPod("pod-a", "microvm-pool")); err != nil { + if _, err := s.syncPod(ctx, workerPod("pod-a", "microvm-pool")); err != nil { t.Fatalf("syncPod(pod-a): %v", err) } if left, err := os.ReadDir(s.scratchDir); err != nil || len(left) != 0 { @@ -575,7 +575,7 @@ func TestSyncPodMicroVMAssetCacheAndEviction(t *testing.T) { storage.mu.Unlock() for _, uid := range []string{"pod-b", "pod-a"} { - if err := s.syncPod(ctx, workerPod(uid, "microvm-pool")); err != nil { + if _, err := s.syncPod(ctx, workerPod(uid, "microvm-pool")); err != nil { t.Fatalf("syncPod(%s) after eviction with bucket offline: %v", uid, err) } got, err := sandboxruntimes.Read(s.runtimesFile(uid)) @@ -593,7 +593,7 @@ func TestSyncPodMicroVMAssetCacheAndEviction(t *testing.T) { // Promoting the evicted v1 to default while the bucket is offline must fail // the sync and keep pod-a's previous file and default-runtime pin in place. cfg.Spec.DefaultVersion = "v1" - if err := s.syncPod(ctx, workerPod("pod-a", "microvm-pool")); err == nil { + if _, err := s.syncPod(ctx, workerPod("pod-a", "microvm-pool")); err == nil { t.Fatal("syncPod(pod-a) after promoting evicted v1 to default with bucket offline = nil, want error") } got, err := sandboxruntimes.Read(s.runtimesFile("pod-a")) @@ -847,7 +847,7 @@ func TestSyncPodAdvertisesWhatItWrote(t *testing.T) { t.Run("a failed sync keeps the entry of the file on disk", func(t *testing.T) { s := newTestSyncer(t, resolvedPool("pool", "gvisor-release"), multiVersionConfig()) - if err := s.syncPod(ctx, pod); err != nil { + if _, err := s.syncPod(ctx, pod); err != nil { t.Fatalf("syncPod: %v", err) } before := advertised(t, s) @@ -860,7 +860,7 @@ func TestSyncPodAdvertisesWhatItWrote(t *testing.T) { s.cpuFeatures = map[string]string{} s.mu.Unlock() s.runCPUFeatures = func(context.Context, string) ([]string, error) { return []string{"not_a_known_feature"}, nil } - if err := s.syncPod(ctx, pod); !errors.Is(err, errPermanent) { + if _, err := s.syncPod(ctx, pod); !errors.Is(err, errPermanent) { t.Fatalf("syncPod with the default's probe failing = %v, want errPermanent", err) } if after := advertised(t, s); after.defaultHash != before.defaultHash || !slices.Equal(after.hashes, before.hashes) { @@ -878,7 +878,7 @@ func TestSyncPodAdvertisesWhatItWrote(t *testing.T) { return map[string]string{gvisorAssetName: "sha-v2"}, nil }, } - if err := s.syncPod(ctx, pod); !errors.Is(err, errIncomplete) { + if _, err := s.syncPod(ctx, pod); !errors.Is(err, errIncomplete) { t.Fatalf("syncPod with a restorable version failing = %v, want errIncomplete", err) } if cr := advertised(t, s); cr.defaultHash != hashOf("v2") || !slices.Equal(cr.hashes, []string{hashOf("v2")}) { diff --git a/docs/api-guide.md b/docs/api-guide.md index 5d98d077b5..a209570dd6 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -86,7 +86,7 @@ worker pod on a GPU node and reserves the device, and nothing else reads it. | :--- | :--- | :--- | | `replicas` | `int32` | Total number of worker pods, mirrored from the managed Deployment. | | `selector` | `string` | Label selector for the worker pods. | -| `sandboxClasses` | `[]WorkerPoolSandboxClassStatus` | One entry per sandbox class, with `name`, `configRef.name` (the `SandboxConfig` in use), `pendingConfigRef.name` (transient, the config the class is switching to; see [Deletion protection](#deletion-protection)), and `conditions`. The `Resolved` condition is `True` (reason `Resolved`) when the class's spec resolves. Otherwise it is `False` with reason `SandboxConfigNotFound`, `SandboxConfigDeleting`, `SandboxClassMismatch`, or `DefaultConfigNotFound`, the controller emits a Warning event, and `configRef` keeps the last resolved config. `configRef` is omitted if the class has never resolved. Each time `configRef` changes, the controller emits a Normal `SandboxConfigSwitched` event naming the old and new config, so `kubectl events --for workerpool/NAME` shows the configs the pool used recently. The API server keeps an event for one hour by default, so the full history is only in a logging backend that stores events. The controller creates the worker Deployment only once every class has a config, and keeps updating an existing Deployment while a class does not resolve. | +| `sandboxClasses` | `[]WorkerPoolSandboxClassStatus` | One entry per sandbox class, with `name`, `configRef.name` (the `SandboxConfig` in use), `pendingConfigRef.name` (transient, the config the class is switching to; see [Deletion protection](#deletion-protection)), and `conditions`. The `Resolved` condition is `True` (reason `Resolved`) when the class's spec resolves. Otherwise it is `False` with reason `SandboxConfigNotFound`, `SandboxConfigDeleting`, `SandboxClassMismatch`, or `DefaultConfigNotFound`, the controller emits a Warning event, and `configRef` keeps the last resolved config. `configRef` is omitted if the class has never resolved. Each time `configRef` changes, the controller emits a Normal `SandboxConfigSwitched` event naming the old and new config, so `kubectl events --for workerpool/NAME` shows the configs the pool used recently. Node-side failures to sync a resolved config's runtimes (for example, an unreachable asset URL or SHA256 mismatch) are emitted by atelet as Warning `SandboxRuntimeSyncFailed` events on the worker pod (`kubectl events --for pod/POD_NAME`). The API server keeps an event for one hour by default, so the full history is only in a logging backend that stores events. The controller creates the worker Deployment only once every class has a config, and keeps updating an existing Deployment while a class does not resolve. | --- diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index 8b03be0cc8..83ce6df1ef 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -44,6 +44,11 @@ rules: - apiGroups: ["ate.dev"] resources: ["workerpools"] verbs: ["get", "list", "watch"] +# Sync failures are reported as Warning Events on the worker pod +# (see cmd/atelet/worker_sandbox_syncer.go). +- apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch"] # ClusterTrustBundles referenced by SystemInfo trustBundle data sources are # resolved on the node: atelet reads them through an informer and projects # the sanitized PEM into actors (see cmd/atelet/trustbundle.go). From 551dd54eef40efa20ce159c20380f51a17776866 Mon Sep 17 00:00:00 2001 From: Yufan Su Date: Tue, 27 Oct 2026 00:29:36 +0000 Subject: [PATCH 03/53] WorkerPools with similar long names no longer share a NetworkPolicy (#3130) Fixes #2934 Builds on #2956 by @ericcurtin, with additional fixes when appling network policy. ### Why `NetworkPolicyName` kept only the first 30 characters of the pool name plus 5 hex digits of its SHA-256. Two pools in the same namespace could get the same NetworkPolicy name. The controller applies with `ForceOwnership`, so the second pool took over the first pool's policy and rewrote its pod selector. ### What changed - The name is now `resources.NetworkPolicyNamePrefix` (`substrate-`) plus the pool name. The pool name is also the `ate.dev/worker-pool` label value, which Kubernetes caps at 63 characters, so the name needs no truncation. On upgrade, each existing pool gets a second policy under the new name. The old one has the same rules and is deleted with the pool. - Before applying, the controller reads the existing policy. It refuses to take it over if another object controls it, or if nothing controls it and its `ate.dev/worker-pool` label isn't the pool's name. A refusal returns an error and emits a `NetworkPolicyConflict` Warning event on the WorkerPool. ### Upgrade The new ate-controller creates `substrate-` for each pool, but leaves the old `substrate--` policy in place until the pool is deleted. Policies add together, so if a later release tightens the rule, the old policy keeps admitting the old peers. If you upgrade in place, wait until the new ate-controller has created `substrate-` for each pool, then delete the old policies: ```sh kubectl get networkpolicy -A -o go-template='{{range .items}}{{$np := .}}{{range .metadata.ownerReferences}}{{if and .controller (eq .kind "WorkerPool") (ne $np.metadata.name (printf "substrate-%s" .name))}}{{$np.metadata.namespace}} {{$np.metadata.name}}{{"\n"}}{{end}}{{end}}{{end}}' | while read -r ns name; do kubectl -n "$ns" delete networkpolicy "$name"; done ``` This deletes only policies a WorkerPool controls whose name isn't `substrate-`. Run the `kubectl get` part alone first to see the list. AI use: AI-assisted (Claude Code). Code reviewed by myself. - [x] Tests pass - [ ] Docs updated (none needed) --------- Signed-off-by: Yufan Su Co-authored-by: Eric Curtin --- .../controllers/networkpolicy_controller.go | 47 ++++++++ .../networkpolicy_controller_test.go | 111 ++++++++++++++++++ .../controllers/workerpool_controller_test.go | 1 + cmd/atecontroller/main.go | 1 + internal/resources/networkpolicy.go | 26 +--- internal/resources/networkpolicy_test.go | 70 ----------- 6 files changed, 165 insertions(+), 91 deletions(-) delete mode 100644 internal/resources/networkpolicy_test.go diff --git a/cmd/atecontroller/internal/controllers/networkpolicy_controller.go b/cmd/atecontroller/internal/controllers/networkpolicy_controller.go index 210539a458..0d67a67d5e 100644 --- a/cmd/atecontroller/internal/controllers/networkpolicy_controller.go +++ b/cmd/atecontroller/internal/controllers/networkpolicy_controller.go @@ -18,11 +18,15 @@ import ( "context" "fmt" + corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" k8errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" + "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -34,6 +38,11 @@ import ( const ( networkPolicyFieldOwner = "ate-networkpolicy" atenetRouterAppName = "atenet-router" + // eventReasonNetworkPolicyConflict is the reason of the Warning event a + // pool emits when checkNetworkPolicyOwner refuses its NetworkPolicy. + eventReasonNetworkPolicyConflict = "NetworkPolicyConflict" + // eventActionApplyNetworkPolicy is the action of that event. + eventActionApplyNetworkPolicy = "ApplyNetworkPolicy" ) type NetworkPolicyReconciler struct { @@ -44,10 +53,15 @@ type NetworkPolicyReconciler struct { // ingress policy admits only that namespace, so a value that does not // match the running router blocks every request to the worker pool. SystemNamespace string + + // Recorder emits a Warning event on the pool when checkNetworkPolicyOwner + // refuses its NetworkPolicy, which leaves the pool without one. + Recorder events.EventRecorder } //+kubebuilder:rbac:groups=ate.dev,resources=workerpools,verbs=get;list;watch //+kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch func (r *NetworkPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) @@ -80,6 +94,19 @@ func (r *NetworkPolicyReconciler) reconcileImpl(ctx context.Context, wp *atev1.W npAC := r.buildNetworkPolicyApplyConfig(wp) + existing := &networkingv1.NetworkPolicy{} + err := r.Get(ctx, types.NamespacedName{Namespace: *npAC.Namespace, Name: *npAC.Name}, existing) + switch { + case k8errors.IsNotFound(err): + case err != nil: + return fmt.Errorf("failed to get NetworkPolicy %s:%s: %w", *npAC.Namespace, *npAC.Name, err) + default: + if err := checkNetworkPolicyOwner(existing, wp); err != nil { + r.Recorder.Eventf(wp, existing, corev1.EventTypeWarning, eventReasonNetworkPolicyConflict, eventActionApplyNetworkPolicy, "%v", err) + return err + } + } + if err := r.Apply(ctx, npAC, client.FieldOwner(networkPolicyFieldOwner), client.ForceOwnership); err != nil { return fmt.Errorf("failed to apply NetworkPolicy %s:%s: %w", *npAC.Namespace, *npAC.Name, err) } @@ -90,6 +117,26 @@ func (r *NetworkPolicyReconciler) reconcileImpl(ctx context.Context, wp *atev1.W return nil } +// checkNetworkPolicyOwner returns an error if np belongs to something other +// than wp. The apply forces ownership, so without this check it would rewrite +// another pool's policy, or a policy someone else created with the same name. +// A policy controlled by wp passes even if its label was edited, so the apply +// restores it. A policy with no controller passes only if it is labeled for wp. +func checkNetworkPolicyOwner(np *networkingv1.NetworkPolicy, wp *atev1.WorkerPool) error { + if owner := metav1.GetControllerOf(np); owner != nil { + if owner.UID == wp.UID { + return nil + } + return fmt.Errorf("refusing to take over NetworkPolicy %s:%s: controlled by %s %s (UID %s)", + np.Namespace, np.Name, owner.Kind, owner.Name, owner.UID) + } + if got := np.Labels[workerPoolLabel]; got != wp.Name { + return fmt.Errorf("refusing to take over NetworkPolicy %s:%s: label %s is %q, want %q", + np.Namespace, np.Name, workerPoolLabel, got, wp.Name) + } + return nil +} + func (r *NetworkPolicyReconciler) buildNetworkPolicyApplyConfig(wp *atev1.WorkerPool) *networkingv1ac.NetworkPolicyApplyConfiguration { np := networkingv1ac.NetworkPolicy(resources.NetworkPolicyName(wp.Name), wp.Namespace). WithLabels(map[string]string{ diff --git a/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go b/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go index 2e07170943..cd43ac2373 100644 --- a/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go +++ b/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go @@ -16,11 +16,18 @@ package controllers import ( "context" + "slices" "testing" networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" + networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" "go.ate.dev/substrate/internal/installdefaults" "go.ate.dev/substrate/internal/resources" ) @@ -112,3 +119,107 @@ func TestBuildNetworkPolicyRelocatedNamespace(t *testing.T) { t.Errorf("ingress namespace selector = %q, want %q", got, relocated) } } + +// TestNetworkPolicyTakeover pre-creates the pool's NetworkPolicy and checks +// that reconcile rewrites it only when it belongs to the pool. +func TestNetworkPolicyTakeover(t *testing.T) { + tests := []struct { + name string + label string + ownerUID types.UID + wantErr bool + }{ + { + name: "labeled for another pool", + label: "other-pool", + wantErr: true, + }, + { + name: "unlabeled", + wantErr: true, + }, + { + name: "controlled by an earlier pool with the same name", + label: "netpol-takeover", + ownerUID: "earlier-uid", + wantErr: true, + }, + { + name: "orphaned policy of this pool", + label: "netpol-takeover", + }, + { + name: "controlled by this pool with an edited label", + label: "edited", + ownerUID: "netpol-takeover-uid", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + // The pool is never created, so only this test reconciles it. + wp := makeWorkerPool("netpol-takeover", "default", 1, "worker:v1") + wp.UID = "netpol-takeover-uid" + + // Apply as the controller's field manager, as an earlier reconcile would. + // The API server rejects a second controller reference written by another + // manager, so the earlier-pool row would pass even without the check. + ac := networkingv1ac.NetworkPolicy(resources.NetworkPolicyName(wp.Name), wp.Namespace). + WithSpec(networkingv1ac.NetworkPolicySpec(). + WithPolicyTypes(networkingv1.PolicyTypeIngress)) + if tt.label != "" { + ac.WithLabels(map[string]string{workerPoolLabel: tt.label}) + } + if tt.ownerUID != "" { + ac.WithOwnerReferences(metav1ac.OwnerReference(). + WithAPIVersion(atev1.GroupVersion.String()). + WithKind("WorkerPool"). + WithName(wp.Name). + WithUID(tt.ownerUID). + WithController(true)) + } + if err := k8sClient.Apply(ctx, ac, client.FieldOwner(networkPolicyFieldOwner)); err != nil { + t.Fatalf("apply NetworkPolicy: %v", err) + } + np := &networkingv1.NetworkPolicy{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: *ac.Name, Namespace: *ac.Namespace}, np); err != nil { + t.Fatalf("get NetworkPolicy: %v", err) + } + deleteOnCleanup(t, np) + + rec := events.NewFakeRecorder(10) + r := &NetworkPolicyReconciler{Client: k8sClient, SystemNamespace: installdefaults.SystemNamespace, Recorder: rec} + err := r.reconcileImpl(ctx, wp) + if (err != nil) != tt.wantErr { + t.Fatalf("reconcileImpl() error = %v, wantErr %v", err, tt.wantErr) + } + var gotEvents, wantEvents []string + for len(rec.Events) > 0 { + gotEvents = append(gotEvents, <-rec.Events) + } + if err != nil { + wantEvents = []string{"Warning " + eventReasonNetworkPolicyConflict + " " + err.Error()} + } + if !slices.Equal(gotEvents, wantEvents) { + t.Errorf("events = %q, want %q", gotEvents, wantEvents) + } + + got := &networkingv1.NetworkPolicy{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(np), got); err != nil { + t.Fatalf("get NetworkPolicy: %v", err) + } + if tt.wantErr { + if got.ResourceVersion != np.ResourceVersion { + t.Errorf("NetworkPolicy was rewritten: %+v", got) + } + return + } + if !metav1.IsControlledBy(got, wp) { + t.Errorf("NetworkPolicy owners = %+v, want controller UID %s", got.OwnerReferences, wp.UID) + } + if got.Labels[workerPoolLabel] != wp.Name { + t.Errorf("NetworkPolicy label %s = %q, want %q", workerPoolLabel, got.Labels[workerPoolLabel], wp.Name) + } + }) + } +} diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index 38395829d6..1213d37b99 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -108,6 +108,7 @@ func TestMain(m *testing.M) { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), SystemNamespace: installdefaults.SystemNamespace, + Recorder: mgr.GetEventRecorder("networkpolicy-controller"), }).SetupWithManager(mgr); err != nil { fmt.Fprintf(os.Stderr, "netpolicy controller setup failed: %v\n", err) os.Exit(1) diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index d264c67226..ec8b240552 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -284,6 +284,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), SystemNamespace: systemNamespace, + Recorder: mgr.GetEventRecorder("networkpolicy-controller"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NetPolicy") os.Exit(1) diff --git a/internal/resources/networkpolicy.go b/internal/resources/networkpolicy.go index 3bccd6a87e..bbd33e6ad4 100644 --- a/internal/resources/networkpolicy.go +++ b/internal/resources/networkpolicy.go @@ -14,26 +14,10 @@ package resources -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "strings" -) - -// NetworkPolicyName returns the deterministic Kubernetes NetworkPolicy name -// generated for a given WorkerPool name. +// NetworkPolicyName returns the NetworkPolicy name for a WorkerPool. It keeps +// the whole pool name, so each pool gets its own policy. The policy also uses +// the pool name as its ate.dev/worker-pool label value, which Kubernetes caps +// at 63 characters, so the result never needs truncating. func NetworkPolicyName(wpName string) string { - sum := sha256.Sum256([]byte(wpName)) - hash := hex.EncodeToString(sum[:]) - if len(hash) > 5 { - hash = hash[:5] - } - truncated := wpName - if len(truncated) > 30 { - truncated = truncated[:30] - } - // Trim trailing hyphens to prevent double hyphens when appending "-". - truncated = strings.TrimRight(truncated, "-") - return fmt.Sprintf("substrate-%s-%s", truncated, hash) + return "substrate-" + wpName } diff --git a/internal/resources/networkpolicy_test.go b/internal/resources/networkpolicy_test.go deleted file mode 100644 index 933b712d33..0000000000 --- a/internal/resources/networkpolicy_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package resources - -import ( - "strings" - "testing" -) - -func TestNetworkPolicyName(t *testing.T) { - tests := []struct { - name string - wpName string - wantPrefix string - }{ - { - name: "short workerpool name", - wpName: "my-wp", - wantPrefix: "substrate-my-wp-", - }, - { - name: "long workerpool name truncated at 30 chars", - wpName: "a-very-long-workerpool-name-that-exceeds-30-chars", - wantPrefix: "substrate-a-very-long-workerpool-name-th-", - }, - { - name: "workerpool name truncated ending with a hyphen", - wpName: "this-is-a-long-worker-pool-n-with-hyphen-at-30", - // "this-is-a-long-worker-pool-n-" is 30 chars long, ending in a hyphen. - // Trimming trailing hyphens turns it into "this-is-a-long-worker-pool-n". - wantPrefix: "substrate-this-is-a-long-worker-pool-n-", - }, - { - name: "workerpool name ending with hyphens", - wpName: "workerpool-name---", - wantPrefix: "substrate-workerpool-name-", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := NetworkPolicyName(tt.wpName) - - if !strings.HasPrefix(got, tt.wantPrefix) { - t.Errorf("NetworkPolicyName(%q) = %q, want prefix %q", tt.wpName, got, tt.wantPrefix) - } - - // Ensure no double hyphens before the hash - // The format is substrate-- - parts := strings.Split(got, "-") - for i, part := range parts { - if part == "" && i > 0 && i < len(parts)-1 { - t.Errorf("NetworkPolicyName(%q) = %q contains empty part (double hyphen)", tt.wpName, got) - } - } - }) - } -} From 28520c0d6795eebe355a645438940dc517e9f351 Mon Sep 17 00:00:00 2001 From: Greg Castle Date: Tue, 27 Oct 2026 01:32:21 +0000 Subject: [PATCH 04/53] Generate the actor identity CA with ECDSA P-256 so external relying parties accept it (#3251) New installs generated an Ed25519 actor identity CA, which many external relying parties don't accept as a trust anchor. - `ate-setup` now generates `actor-id-ca-pool` with ECDSA P-256, the same way it already generated `egress-mitm-ca-pool`. Cluster-internal pools (pod identity, service DNS, postgres) stay Ed25519. - `kubectl ate admin make-ca-pool --key-type` now defaults to `ECDSAP256`. Pass `--key-type ED25519` for pools that are only validated inside the cluster. - Existing pools are kept on upgrade, so installed clusters keep their current root until it is rotated. AI use: AI-assisted, I reviewed and tested it. - [x] Tests pass - [x] Docs: I don't think this needs a docs change since existing installs are unchanged. Signed-off-by: Greg Castle --- cmd/ate-setup/internal/steps/create.go | 16 +++++++------ cmd/ate-setup/internal/steps/create_test.go | 25 ++++++++++++++++++++- cmd/kubectl-ate/internal/cmd/admin.go | 25 +++++++++++++-------- cmd/kubectl-ate/internal/cmd/admin_test.go | 13 +++++++++++ 4 files changed, 62 insertions(+), 17 deletions(-) diff --git a/cmd/ate-setup/internal/steps/create.go b/cmd/ate-setup/internal/steps/create.go index 705365b25e..296a450cb5 100644 --- a/cmd/ate-setup/internal/steps/create.go +++ b/cmd/ate-setup/internal/steps/create.go @@ -63,10 +63,12 @@ func (e *Env) CreateJWTAuthorityPoolSecret(ctx context.Context) error { return e.createJWTPool(ctx, e.Namespace(), SecretActorIDJWTPool) } -// CreateActorIDCAPoolSecret generates the actor-identity CA pool. +// CreateActorIDCAPoolSecret generates the actor-identity CA pool. External +// relying parties trust this root, so it uses ECDSA P-256, which is more +// broadly accepted than Ed25519. func (e *Env) CreateActorIDCAPoolSecret(ctx context.Context) error { log.Step("create_actor_id_ca_pool_secret") - return e.createCAPool(ctx, e.Namespace(), SecretActorIDCAPool) + return e.createCAPool(ctx, e.Namespace(), SecretActorIDCAPool, localca.KeyTypeECDSAP256) } // CreateEgressMITMCAPoolSecret generates the egress MITM CA pool. @@ -102,13 +104,13 @@ func (e *Env) CreatePodCertificateControllerCAs(ctx context.Context) error { if err := e.Kube.EnsureNamespace(ctx, NamespacePodCert); err != nil { return err } - if err := e.createCAPool(ctx, NamespacePodCert, SecretServiceDNSCA); err != nil { + if err := e.createCAPool(ctx, NamespacePodCert, SecretServiceDNSCA, localca.KeyTypeED25519); err != nil { return err } - if err := e.createCAPool(ctx, NamespacePodCert, SecretPodIdentityCA); err != nil { + if err := e.createCAPool(ctx, NamespacePodCert, SecretPodIdentityCA, localca.KeyTypeED25519); err != nil { return err } - return e.createCAPool(ctx, NamespacePodCert, SecretPostgresCA) + return e.createCAPool(ctx, NamespacePodCert, SecretPostgresCA, localca.KeyTypeED25519) } // CreateActorIDCACertsSecret derives a certificate-only trust bundle from the @@ -238,7 +240,7 @@ func buildAuthenticationConfig(issuer string) string { // An existing pool is left alone. Regenerating it would rotate the root out // from under every certificate already issued from it, and the callers in the // shell script all guarded these calls with an existence check for that reason. -func (e *Env) createCAPool(ctx context.Context, namespace, name string) error { +func (e *Env) createCAPool(ctx context.Context, namespace, name string, keyType localca.KeyType) error { exists, err := e.Kube.SecretExists(ctx, namespace, name) if err != nil { return err @@ -248,7 +250,7 @@ func (e *Env) createCAPool(ctx context.Context, namespace, name string) error { return nil } - data, err := newCAPoolSecretData(poolKeyID, localca.KeyTypeED25519) + data, err := newCAPoolSecretData(poolKeyID, keyType) if err != nil { return fmt.Errorf("while generating the CA pool for %s/%s: %w", namespace, name, err) } diff --git a/cmd/ate-setup/internal/steps/create_test.go b/cmd/ate-setup/internal/steps/create_test.go index 0cb3ece7e8..64ca78fb53 100644 --- a/cmd/ate-setup/internal/steps/create_test.go +++ b/cmd/ate-setup/internal/steps/create_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" + "go.ate.dev/substrate/cmd/ate-setup/internal/config" "go.ate.dev/substrate/cmd/ate-setup/internal/kube" "go.ate.dev/substrate/internal/localca" "go.ate.dev/substrate/internal/localjwtauthority" @@ -112,7 +113,8 @@ func TestNewCAPoolSecretData(t *testing.T) { keyType localca.KeyType wantAlg x509.PublicKeyAlgorithm }{ - {"actor identity", poolKeyID, localca.KeyTypeED25519, x509.Ed25519}, + {"pod certificates", poolKeyID, localca.KeyTypeED25519, x509.Ed25519}, + {"actor identity", poolKeyID, localca.KeyTypeECDSAP256, x509.ECDSA}, {"egress mitm", poolKeyID, localca.KeyTypeECDSAP256, x509.ECDSA}, } { t.Run(tc.name, func(t *testing.T) { @@ -165,6 +167,27 @@ func TestNewCAPoolSecretData(t *testing.T) { } } +// External relying parties trust the actor identity root, and many accept +// ECDSA but not Ed25519. +func TestCreateActorIDCAPoolSecretIsECDSA(t *testing.T) { + ctx := context.Background() + e := &Env{Cfg: &config.Config{}, Kube: fakeKube(t)} + if err := e.CreateActorIDCAPoolSecret(ctx); err != nil { + t.Fatalf("CreateActorIDCAPoolSecret() error = %v", err) + } + secret, err := e.Kube.Typed.CoreV1().Secrets(e.Namespace()).Get(ctx, SecretActorIDCAPool, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Get(%s) error = %v", SecretActorIDCAPool, err) + } + pool, err := localca.Unmarshal(secret.Data["pool"]) + if err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if got := pool.CAs[0].RootCertificate.PublicKeyAlgorithm; got != x509.ECDSA { + t.Errorf("root key algorithm = %v, want %v", got, x509.ECDSA) + } +} + func TestNewJWTPoolSecretData(t *testing.T) { for _, alg := range []string{"ES256", "RS256"} { t.Run(alg, func(t *testing.T) { diff --git a/cmd/kubectl-ate/internal/cmd/admin.go b/cmd/kubectl-ate/internal/cmd/admin.go index 8b73cd227b..ac644ea14e 100644 --- a/cmd/kubectl-ate/internal/cmd/admin.go +++ b/cmd/kubectl-ate/internal/cmd/admin.go @@ -54,14 +54,9 @@ var makeCaPoolCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - var keyType localca.KeyType - switch makeCaPoolKeyTypeFlag { - case "ED25519": - keyType = localca.KeyTypeED25519 - case "ECDSAP256": - keyType = localca.KeyTypeECDSAP256 - default: - return usageErrorf("unknown key type %q", makeCaPoolKeyTypeFlag) + keyType, err := parseCAKeyType(makeCaPoolKeyTypeFlag) + if err != nil { + return err } ca, err := localca.GenerateCA( @@ -310,6 +305,18 @@ func printJWTKeys(out io.Writer, pool *localjwtauthority.ConcretePool) error { return w.Flush() } +// parseCAKeyType maps a make-ca-pool --key-type value to a localca.KeyType. +func parseCAKeyType(keyType string) (localca.KeyType, error) { + switch keyType { + case "ECDSAP256": + return localca.KeyTypeECDSAP256, nil + case "ED25519": + return localca.KeyTypeED25519, nil + default: + return 0, usageErrorf("unknown key type %q", keyType) + } +} + // newKubeClient builds a client from the --kubeconfig and --context flags. func newKubeClient(errOut io.Writer) (kubernetes.Interface, error) { kconfig, err := loadKubeConfig(errOut, kubeconfig, k8sContext) @@ -354,7 +361,7 @@ func init() { makeCaPoolCmd.Flags().StringVar(&makeCaPoolIDFlag, "ca-id", "", "ID of the initial CA in the pool") makeCaPoolCmd.Flags().StringVar(&poolSecretNamespaceFlag, "secret-namespace", "default", "namespace of the pool secret") makeCaPoolCmd.Flags().StringVar(&poolSecretNameFlag, "name", "", "name of the pool secret") - makeCaPoolCmd.Flags().StringVar(&makeCaPoolKeyTypeFlag, "key-type", "ED25519", "CA key type: ED25519 or ECDSAP256") + makeCaPoolCmd.Flags().StringVar(&makeCaPoolKeyTypeFlag, "key-type", "ECDSAP256", "CA key type: ECDSAP256 or ED25519. ECDSAP256 is more broadly accepted by external relying parties") makeCaPoolCmd.Flags().DurationVar(&makeCaPoolValidityFlag, "validity", 365*24*time.Hour, "how long the CA certificate is valid for") adminCmd.AddCommand(makeCaPoolCmd) diff --git a/cmd/kubectl-ate/internal/cmd/admin_test.go b/cmd/kubectl-ate/internal/cmd/admin_test.go index 2cab7ec11b..a42b186648 100644 --- a/cmd/kubectl-ate/internal/cmd/admin_test.go +++ b/cmd/kubectl-ate/internal/cmd/admin_test.go @@ -30,9 +30,22 @@ import ( "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" + "go.ate.dev/substrate/internal/localca" "go.ate.dev/substrate/internal/localjwtauthority" ) +// The default must suit a root trusted by external relying parties, and many +// accept ECDSA but not Ed25519. +func TestMakeCAPoolDefaultKeyTypeIsECDSA(t *testing.T) { + got, err := parseCAKeyType(makeCaPoolCmd.Flags().Lookup("key-type").DefValue) + if err != nil { + t.Fatal(err) + } + if got != localca.KeyTypeECDSAP256 { + t.Errorf("default key type = %v, want KeyTypeECDSAP256", got) + } +} + func TestNewJWTPoolSecretWithFlagDefaults(t *testing.T) { alg := makeJwtPoolCmd.Flags().Lookup("alg").DefValue keyID := makeJwtPoolCmd.Flags().Lookup("key-id").DefValue From 006f6dc2b55b5ac5251a39f373dfd0a680e81fc5 Mon Sep 17 00:00:00 2001 From: Grant McCloskey Date: Tue, 27 Oct 2026 03:06:46 +0000 Subject: [PATCH 05/53] Releases publish the Kubernetes Secrets credential provider image (#3220) Kind installs select the bundled Kubernetes Secrets credential provider, but releases do not publish its image, so `ate-setup deploy --image-repo ghcr.io/agent-substrate/substrate --credential-provider='{"name":"k8s.io"}'` fails. This adds it to `released`. Part of #3217. envoy-dataplane, the other image #3217 needs, is left for a separate PR because the release job has no buildx. After the first release that includes it, an org owner has to make the new `ghcr.io/agent-substrate/substrate/kubernetes-secrets` package public, as `docs/dev/releasing.md` describes. AI use: I used Claude Code to find the affected tests and docs and to draft the change. I reviewed the diff and ran the ate-setup tests. - [x] Tests pass - [x] Docs updated Signed-off-by: Grant McCloskey --- cmd/ate-setup/internal/images/images.go | 1 + cmd/ate-setup/internal/images/prebuilt_test.go | 6 +++--- cmd/ate-setup/internal/steps/publish_test.go | 7 +++++-- docs/dev/releasing.md | 4 ++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cmd/ate-setup/internal/images/images.go b/cmd/ate-setup/internal/images/images.go index 2110fe26b9..56944d9ad0 100644 --- a/cmd/ate-setup/internal/images/images.go +++ b/cmd/ate-setup/internal/images/images.go @@ -72,6 +72,7 @@ var released = []string{ "cmd/atenet", "cmd/ateworker-gvisor", "cmd/ateworker-microvm", + "cmd/credential-provider/kubernetes-secrets", "cmd/objectstore-plugin", "cmd/podcertcontroller", } diff --git a/cmd/ate-setup/internal/images/prebuilt_test.go b/cmd/ate-setup/internal/images/prebuilt_test.go index ccf5bc48ce..7fa604d3ba 100644 --- a/cmd/ate-setup/internal/images/prebuilt_test.go +++ b/cmd/ate-setup/internal/images/prebuilt_test.go @@ -232,15 +232,15 @@ func TestPrebuiltReportsEveryFailure(t *testing.T) { // such as one the operator published to. When the lookup fails, the error says // releases do not publish it; for a released image it does not. func TestPrebuiltUnreleasedImages(t *testing.T) { - const ref = "image: ko://go.ate.dev/substrate/cmd/credential-provider/kubernetes-secrets\n" - const hint = "releases do not publish kubernetes-secrets" + const ref = "image: ko://go.ate.dev/substrate/demos/counter\n" + const hint = "releases do not publish counter" got, err := images.NewPrebuilt(testSource, newStubRegistry().digest). ResolveBytes(context.Background(), []byte(ref)) if err != nil { t.Fatalf("ResolveBytes() from a registry that holds it = %v", err) } - if want := "image: example.com/substrate/kubernetes-secrets:v1.2.3" + digestSuffix + "\n"; string(got) != want { + if want := "image: example.com/substrate/counter:v1.2.3" + digestSuffix + "\n"; string(got) != want { t.Errorf("ResolveBytes() = %q, want %q", got, want) } diff --git a/cmd/ate-setup/internal/steps/publish_test.go b/cmd/ate-setup/internal/steps/publish_test.go index 90288247a9..96a6c59fbe 100644 --- a/cmd/ate-setup/internal/steps/publish_test.go +++ b/cmd/ate-setup/internal/steps/publish_test.go @@ -128,14 +128,17 @@ func TestReleaseImageRefs(t *testing.T) { if want := len(images.ReleaseComponents()); len(refs) != want { t.Fatalf("releaseImageRefs() returned %d refs, want %d", len(refs), want) } - for _, want := range []string{"example.com/substrate/ateapi:v1", "example.com/substrate/objectstore-plugin:v1"} { + for _, want := range []string{ + "example.com/substrate/ateapi:v1", + "example.com/substrate/kubernetes-secrets:v1", + "example.com/substrate/objectstore-plugin:v1", + } { if !slices.Contains(refs, want) { t.Errorf("releaseImageRefs() = %v, want %s", refs, want) } } for _, unwanted := range []string{ "example.com/substrate/counter:v1", - "example.com/substrate/kubernetes-secrets:v1", "example.com/substrate/envoy-dataplane:v1", } { if slices.Contains(refs, unwanted) { diff --git a/docs/dev/releasing.md b/docs/dev/releasing.md index 96e9a5893d..029e17f744 100644 --- a/docs/dev/releasing.md +++ b/docs/dev/releasing.md @@ -37,11 +37,11 @@ To preview the notes before tagging, run the dry run with the tag you plan to pu ## Release images -A release publishes `ate-idp-server`, `ateapi`, `atecontroller`, `atelet`, `atenet`, `ateworker-gvisor`, `ateworker-microvm`, `objectstore-plugin`, and `podcertcontroller`, as `ghcr.io/agent-substrate/substrate/:` (`images.ReleaseComponents` in [`cmd/ate-setup/internal/images`](../../cmd/ate-setup/internal/images/images.go)). The object storage plugin ships with releases until it moves out of this repository. +A release publishes `ate-idp-server`, `ateapi`, `atecontroller`, `atelet`, `atenet`, `ateworker-gvisor`, `ateworker-microvm`, `kubernetes-secrets` (the Kubernetes Secrets credential provider), `objectstore-plugin`, and `podcertcontroller`, as `ghcr.io/agent-substrate/substrate/:` (`images.ReleaseComponents` in [`cmd/ate-setup/internal/images`](../../cmd/ate-setup/internal/images/images.go)). The object storage plugin ships with releases until it moves out of this repository. The list is an allowlist, `released` in `images.go`. A new component is not published until a pull request adds it there, so publishing an image is always an explicit, reviewed change. -It leaves out `envoy-dataplane`, the Kubernetes Secrets credential provider, the demos, and the benchmarking fakes. The release images are therefore not a complete install of the default configuration, and the notes give no install command: an install has to take the images releases leave out from another registry, or build them from source. `ate-setup --image-repo` looks those images up like any other; if one is missing, the error says releases do not publish it. +It leaves out `envoy-dataplane`, the demos, and the benchmarking fakes. The release images are therefore not a complete install of the default configuration, and the notes give no install command: an install has to take the images releases leave out from another registry, or build them from source. `ate-setup --image-repo` looks those images up like any other; if one is missing, the error says releases do not publish it. Tags that begin `vX.Y.Z` start the workflow. `ate-setup` refuses any version that is not `vX.Y.Z` or `vX.Y.Z-PRERELEASE` before building, so a tag like `v1.2.3foo` publishes nothing. From a4f92ae80f62369d29766473078323d8b37ac0b4 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 27 Oct 2026 03:15:41 +0000 Subject: [PATCH 06/53] docs: require DCO sign-off instead of the Google CLA (#3237) Why is this needed? (Or link the issue. Open one first for features, API, install or CI changes.) Part of https://github.com/cncf/sandbox/issues/542 I have enabled DCO enforcement, and will shortly drop the Google CLA enforcement (I wanted to get these docs up and double check the DCO is working as expected). AI use: Yes, drafted by AI. I reviewed it. - [x] Tests pass - [x] Docs updated Signed-off-by: Benjamin Elder --- AGENTS.md | 5 +++-- CONTRIBUTING.md | 31 +++++++++++++++++++++---------- GOVERNANCE.md | 10 +++++----- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 75833863d9..269a52b78f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,8 +2,9 @@ ## Rules for contributors -- Read the "Using AI" section of `COLLABORATING.md`. You are responsible for everything your agent submits. -- Say in the PR description that you used AI, and how. Don't add AI co-author or assisted-by trailers. Submit commits and PRs from your own account. We can't accept contributions from agent or bot accounts (a human must sign the contribution agreement). +- Read `CONTRIBUTING.md` and the "Using AI" section of `COLLABORATING.md`. You are responsible for everything your agent submits. +- Say in the PR description that you used AI, and how. Don't add AI co-author or assisted-by trailers. Submit commits and PRs from your own account. We can't accept contributions from agent or bot accounts. +- Agents must never sign off a commit as themselves. Only a human can certify the [DCO](https://developercertificate.org/). - Before asking for review, review your own diff. For substantial changes, also have a fresh agent session review it adversarially, then validate its findings and fix the confirmed issues. - **Be brief.** Code comments, commit messages, PR descriptions and issues should be short, plain English, and read the way an experienced maintainer writes by hand. Reviewer time and agent context windows are limited. - `COLLABORATING.md` doesn't allow AI-generated commit messages, because they have usually been low quality and unreviewed. If you are asked to write one anyway, hold it to the same standard, and the person submitting it must review it. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6b716b883f..022d65cb3e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,19 +6,30 @@ guidelines. ## Before you begin -### Sign our Contributor License Agreement +### Sign off your commits -Contributions to this project must be accompanied by a -[Contributor License Agreement](https://cla.developers.google.com/about) (CLA). -You (or your employer) retain the copyright to your contribution; this simply -gives us permission to use and redistribute your contributions as part of the -project. +Every commit must be signed off under the +[Developer Certificate of Origin](https://developercertificate.org/) (DCO). +The `Signed-off-by` line certifies that you wrote the change or otherwise have +the right to submit it under the project's license. + +Sign off with your real name, not an anonymous ID or a false name. It doesn't +need to be your legal name, just the name the community knows you by. See the +[CNCF DCO guidelines](https://github.com/cncf/foundation/blob/main/policies-guidance/dco-guidelines.md). + +Git adds the line with `-s`, using `user.name` and `user.email` from your git +config. The name and email must match the commit author. + +```sh +git commit -s -m "fix typo in README" +``` -If you or your current employer have already signed the Google CLA (even if it -was for a different project), you probably don't need to do it again. +The DCO check on your PR fails if any commit is missing a sign-off. To sign off +the commits already on your branch, then force-push: -Visit to see your current agreements or to -sign a new one. +```sh +git rebase --signoff upstream/main +``` ### Review our Community Guidelines diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 6efe908911..d3a3dbdff3 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -20,16 +20,16 @@ decided among current Maintainers. **Default** — No formal designation. Anyone may open issues and PRs on the project, or review other people's PRs and issues. Must follow the Code of -Conduct. Opening a PR requires that the author sign the [Google Contributor -License Agreement](https://cla.developers.google.com/about) (see +Conduct. Every commit in a PR must be signed off under the [Developer +Certificate of Origin](https://developercertificate.org/) (see [CONTRIBUTING.md](CONTRIBUTING.md)). **Contributor** — Someone who has made at least one non-trivial contribution and is known to or vouched for by a Maintainer, or has accumulated enough contributions over time. Granted `Read` access on the repository. Must follow -the Code of Conduct and sign the [Google Contributor License -Agreement](https://cla.developers.google.com/about) before their first -contribution is merged (see [CONTRIBUTING.md](CONTRIBUTING.md)). +the Code of Conduct and sign off every commit under the [Developer Certificate +of Origin](https://developercertificate.org/) (see +[CONTRIBUTING.md](CONTRIBUTING.md)). **Reviewer** — A contributor who has made enough contributions and is vouched for by at least 2 Maintainers. Granted `Triage` access: can review, label, and From 451b03094e58f4d9b379b228b7d410301eb448d4 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Tue, 27 Oct 2026 03:23:47 +0000 Subject: [PATCH 07/53] Fix flake: retry adding the kill container on a conflict (#3183) `TestSandboxCrashIsReported` flaked a few times recently. Each time `UpdateEphemeralContainers` returned a 409 Conflict. This PR resolves the image first and retries the get and update on a conflict to deflake. AI assisted and human reviewed. - [ ] Tests pass - [ ] Docs updated --------- Signed-off-by: Haven Xia --- internal/e2e/sandboxkill.go | 47 +++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/internal/e2e/sandboxkill.go b/internal/e2e/sandboxkill.go index 4ec51c02b9..6c0c0fe916 100644 --- a/internal/e2e/sandboxkill.go +++ b/internal/e2e/sandboxkill.go @@ -25,6 +25,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" ) @@ -49,29 +50,35 @@ func testserverImage(t *testing.T) string { // crash would, from an ephemeral container sharing the worker's PID namespace. func KillActorSandbox(ctx context.Context, t *testing.T, clients *Clients, namespace, podName, actorUID string) { t.Helper() + image := testserverImage(t) pods := clients.K8s.CoreV1().Pods(namespace) - pod, err := pods.Get(ctx, podName, metav1.GetOptions{}) - if err != nil { - t.Fatalf("getting worker pod %s/%s: %v", namespace, podName, err) - } - name := fmt.Sprintf("kill-sandbox-%d", len(pod.Spec.EphemeralContainers)) - pod.Spec.EphemeralContainers = append(pod.Spec.EphemeralContainers, corev1.EphemeralContainer{ - TargetContainerName: "worker", - EphemeralContainerCommon: corev1.EphemeralContainerCommon{ - Name: name, - Image: testserverImage(t), - // kills the runsc sandbox and gofer, or cloud-hypervisor and - // virtiofsd: each names the actor UID in its command line - Args: []string{"kill", "--cmdline-contains", actorUID}, - // root, like the sandbox processes, which also run unconfined, and - // the default AppArmor profile, enforced on GKE, may not signal them - SecurityContext: &corev1.SecurityContext{ - RunAsUser: ptr.To[int64](0), - AppArmorProfile: &corev1.AppArmorProfile{Type: corev1.AppArmorProfileTypeUnconfined}, + var name string + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + pod, err := pods.Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return err + } + name = fmt.Sprintf("kill-sandbox-%d", len(pod.Spec.EphemeralContainers)) + pod.Spec.EphemeralContainers = append(pod.Spec.EphemeralContainers, corev1.EphemeralContainer{ + TargetContainerName: "worker", + EphemeralContainerCommon: corev1.EphemeralContainerCommon{ + Name: name, + Image: image, + // kills the runsc sandbox and gofer, or cloud-hypervisor and + // virtiofsd: each names the actor UID in its command line + Args: []string{"kill", "--cmdline-contains", actorUID}, + // root, like the sandbox processes, which also run unconfined, and + // the default AppArmor profile, enforced on GKE, may not signal them + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To[int64](0), + AppArmorProfile: &corev1.AppArmorProfile{Type: corev1.AppArmorProfileTypeUnconfined}, + }, }, - }, + }) + _, err = pods.UpdateEphemeralContainers(ctx, podName, pod, metav1.UpdateOptions{}) + return err }) - if _, err := pods.UpdateEphemeralContainers(ctx, podName, pod, metav1.UpdateOptions{}); err != nil { + if err != nil { t.Fatalf("adding ephemeral container to %s/%s: %v", namespace, podName, err) } From b0202404b159acbeb8756a5b73f5389e6d49dbcb Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Tue, 27 Oct 2026 03:52:54 +0000 Subject: [PATCH 08/53] docs: fix the README note on enabling beta APIs (#3266) The GKE quickstart in `README.md` says the required Kubernetes beta APIs can only be enabled at cluster creation. The `setup-gcp` README says otherwise, and `setup-gcp create cluster` enables them on an existing cluster. Fixes #3233 `README.md` now says GKE 1.37 and higher serve the APIs by default. On 1.36, one can enable the beta APIs and then replace any 1.36 node pools created before that. - [ ] Tests pass - [x] Docs updated Signed-off-by: Haven Xia --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 22ec50d785..a6e0f654b4 100644 --- a/README.md +++ b/README.md @@ -172,8 +172,10 @@ curl -X POST \ that ate-api-server and atelet depend on. See [what `create iam` actually grants](tools/setup-gcp/README.md#what-create-iam-actually-grants) to audit them or apply them manually. If you bring your own cluster instead, - note the required Kubernetes beta APIs can only be enabled **at cluster - creation** — see the [Create Cluster warning](tools/setup-gcp/README.md#2-create-cluster). + it needs the PodCertificateRequest and ClusterTrustBundle APIs, which GKE + 1.37 and higher serve by default. On 1.36, enable them as beta APIs, then + replace any 1.36 node pools created before that. See the + [Create Cluster warning](tools/setup-gcp/README.md#2-create-cluster). 4. Deploy the Agent Substrate system to your cluster: ```bash From 635c521cc682e312d01008f3aa2912588d394938 Mon Sep 17 00:00:00 2001 From: Shruti Nair Date: Tue, 27 Oct 2026 04:22:52 +0000 Subject: [PATCH 09/53] ateapi: deny streaming RPCs that have no authorization rule (#3262) `ate-api-server` installed the authz interceptor only for unary RPCs. Streams passed after authentication alone, so the docs' "an RPC with no rule is denied" didn't hold for them. In the bug bash, a ServiceAccount with no grants could list and describe every service through gRPC reflection. This adds `authz.StreamServerInterceptor`, which denies every stream not in `streamRules`. `ateapi.proto` already can't define streaming RPCs (`TestDefaultRPCPermissions_CoversEveryServedRPC`), so the allowlist holds only gRPC reflection. Reflection stays open to any authenticated caller, the same way Kubernetes discovery is open to `system:authenticated`, since it returns only the schema that's public in `api/ateapi/v1/ateapi.proto`. Unauthenticated callers still get `Unauthenticated` from `apiauthn`, which runs first. Fixes #3231 I used an AI agent (Jetski) to find the gap during the bug bash, draft the change, and run the tests. I reviewed the diff. Signed-off-by: Shruti Nair --- cmd/ateapi/internal/authz/interceptor.go | 23 +++++++++++++ cmd/ateapi/internal/authz/interceptor_test.go | 33 +++++++++++++++++++ cmd/ateapi/internal/authz/registry_test.go | 6 ++-- cmd/ateapi/main.go | 1 + docs/authorization.md | 3 ++ 5 files changed, 63 insertions(+), 3 deletions(-) diff --git a/cmd/ateapi/internal/authz/interceptor.go b/cmd/ateapi/internal/authz/interceptor.go index 586c16f35e..e950bf68c8 100644 --- a/cmd/ateapi/internal/authz/interceptor.go +++ b/cmd/ateapi/internal/authz/interceptor.go @@ -20,6 +20,8 @@ import ( "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/principal" "google.golang.org/grpc" + "google.golang.org/grpc/reflection/grpc_reflection_v1" + "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" ) // UnaryServerInterceptor returns a gRPC unary interceptor that enforces per-RPC @@ -56,3 +58,24 @@ func UnaryServerInterceptor(authorizer *Authorizer) grpc.UnaryServerInterceptor return handler(ctx, req) } } + +// streamRules lists the streaming RPCs ate-api-server serves; any other stream +// is denied. ateapi.proto defines none (see +// TestDefaultRPCPermissions_CoversEveryServedRPC). gRPC reflection is open to +// any authenticated caller, like Kubernetes API discovery: it returns only the +// served API schema, which is public in api/ateapi/v1/ateapi.proto. +var streamRules = map[string]bool{ + grpc_reflection_v1.ServerReflection_ServerReflectionInfo_FullMethodName: true, + grpc_reflection_v1alpha.ServerReflection_ServerReflectionInfo_FullMethodName: true, +} + +// StreamServerInterceptor returns a gRPC stream interceptor that denies every +// streaming RPC not in streamRules. +func StreamServerInterceptor() grpc.StreamServerInterceptor { + return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + if !streamRules[info.FullMethod] { + return apierror.PermissionDenied("permission denied: no authorization rule for %s", info.FullMethod) + } + return handler(srv, ss) + } +} diff --git a/cmd/ateapi/internal/authz/interceptor_test.go b/cmd/ateapi/internal/authz/interceptor_test.go index 33b140000b..110ef6bf1d 100644 --- a/cmd/ateapi/internal/authz/interceptor_test.go +++ b/cmd/ateapi/internal/authz/interceptor_test.go @@ -23,6 +23,8 @@ import ( "go.ate.dev/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/reflection/grpc_reflection_v1" + "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" ) func setupTestAuthorizer(t *testing.T) *Authorizer { @@ -235,6 +237,37 @@ func TestUnaryServerInterceptor_UnregisteredAndHandlerAuthorizedRPCs(t *testing. } } +// Reflection streams pass through; any other stream is denied before its +// handler runs. +func TestStreamServerInterceptor(t *testing.T) { + tests := []struct { + name string + fullMethod string + wantCode codes.Code + }{ + {"reflection v1 passes", grpc_reflection_v1.ServerReflection_ServerReflectionInfo_FullMethodName, codes.OK}, + {"reflection v1alpha passes", grpc_reflection_v1alpha.ServerReflection_ServerReflectionInfo_FullMethodName, codes.OK}, + {"unregistered stream is denied", "/substrate.ateapi.v1.Control/NotARealMethod", codes.PermissionDenied}, + {"unary RPC called as a stream is denied", ateapi.Control_ListAtespaces_FullMethodName, codes.PermissionDenied}, + {"handler-authorized RPC called as a stream is denied", ateapi.WorkerService_RegisterWorker_FullMethodName, codes.PermissionDenied}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + handlerCalled := false + err := StreamServerInterceptor()(nil, nil, &grpc.StreamServerInfo{FullMethod: tc.fullMethod}, func(any, grpc.ServerStream) error { + handlerCalled = true + return nil + }) + if apierror.Code(err) != tc.wantCode { + t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", apierror.Code(err), tc.wantCode, err) + } + if wantHandler := tc.wantCode == codes.OK; handlerCalled != wantHandler { + t.Fatalf("handlerCalled = %v, want %v", handlerCalled, wantHandler) + } + }) + } +} + func TestUnaryServerInterceptor_ActorAndTemplateChecks(t *testing.T) { ctx := context.Background() pool := startPostgres(t) diff --git a/cmd/ateapi/internal/authz/registry_test.go b/cmd/ateapi/internal/authz/registry_test.go index 53487c2797..57d2b6bc77 100644 --- a/cmd/ateapi/internal/authz/registry_test.go +++ b/cmd/ateapi/internal/authz/registry_test.go @@ -692,9 +692,9 @@ func TestDefaultRPCPermissions_CoversEveryServedRPC(t *testing.T) { m := methods.Get(j) fullMethod := "/" + string(sd.FullName()) + "/" + string(m.Name()) if m.IsStreamingClient() || m.IsStreamingServer() { - // The authz interceptor is unary-only, so a streaming RPC would go - // unchecked. - t.Errorf("%s is a streaming RPC, which the authz interceptor does not check", fullMethod) + // StreamServerInterceptor denies any stream not in streamRules, + // and has no per-request checks like defaultRPCPermissions. + t.Errorf("%s is a streaming RPC, which StreamServerInterceptor denies", fullMethod) } served[fullMethod] = true } diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index da71b00d13..8e9d74d6f8 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -399,6 +399,7 @@ func main() { grpc.ChainUnaryInterceptor(unaryInterceptors...), grpc.ChainStreamInterceptor( apiauthn.StreamServerInterceptor(authCfg), + authz.StreamServerInterceptor(), preview.StreamServerInterceptor, ), ) diff --git a/docs/authorization.md b/docs/authorization.md index 271123faa9..5de12f020a 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -229,6 +229,9 @@ certificate. - An RPC with no rule is denied, and a test fails the build if a served RPC has none. +- gRPC reflection is open to any authenticated caller, like Kubernetes API + discovery, since it returns only the API schema, which is public in + `api/ateapi/v1/ateapi.proto`. Any other streaming RPC is denied. - An unauthenticated caller gets `Unauthenticated` before its request is read. - A missing or invalid resource name returns `InvalidArgument`; no check is skipped. From a11540ae7ae01f647e9aea6c35443845fbdcf118 Mon Sep 17 00:00:00 2001 From: Luiz Oliveira Date: Tue, 27 Oct 2026 15:10:50 +0000 Subject: [PATCH 10/53] Rename `SnapshotStorage.status` to `state` (#3215) To match the other lifecycle enums Breaking change: `SnapshotStorage.status` is renamed to `state`, and the `SnapshotStorageStatus` enum to `SnapshotStorageState`. On upgrade: - Generated clients: rename `SnapshotStorageStatus` to `SnapshotStorageState`, `SNAPSHOT_STORAGE_STATUS_*` to `SNAPSHOT_STORAGE_STATE_*`, and in Go, `.Status`/`GetStatus()` on `SnapshotStorage` to `.State`/`GetState()`. - JSON consumers, such as scripts that parse `kubectl ate get -o json`: read `durableSnapshot.state` instead of `durableSnapshot.status`, and expect `SNAPSHOT_STORAGE_STATE_PENDING` and `SNAPSHOT_STORAGE_STATE_COMPLETED`. Stored actors and tags need no migration. Signed-off-by: Luiz Oliveira --- api/ateapi/v1/ateapi.pb.go | 76 +++++++++--------- api/ateapi/v1/ateapi.proto | 18 ++--- .../internal/apivalidation/validation_test.go | 24 +++--- .../apivalidation/zz_generated.validation.go | 10 +-- cmd/ateapi/internal/controlapi/actor.go | 2 +- .../controlapi/actor_template_test.go | 2 +- cmd/ateapi/internal/controlapi/actor_test.go | 18 ++--- cmd/ateapi/internal/controlapi/crash_test.go | 6 +- cmd/ateapi/internal/controlapi/export_test.go | 2 +- .../controlapi/functionaltest/actor_test.go | 18 ++--- .../controlapi/functionaltest/common_test.go | 6 +- .../internal/controlapi/hosting_test.go | 14 ++-- .../internal/controlapi/snapshot_helpers.go | 18 ++--- .../controlapi/snapshot_helpers_test.go | 58 +++++++------- .../internal/controlapi/snapshot_report.go | 6 +- .../controlapi/snapshot_report_test.go | 52 ++++++------ cmd/ateapi/internal/controlapi/tag.go | 2 +- cmd/ateapi/internal/controlapi/tag_test.go | 10 +-- .../controlapi/template_reconciler.go | 6 +- .../controlapi/template_reconciler_test.go | 8 +- .../internal/controlapi/workflow_delete.go | 4 +- .../controlapi/workflow_delete_test.go | 18 ++--- .../internal/controlapi/workflow_resume.go | 4 +- .../controlapi/workflow_resume_detach.go | 4 +- .../controlapi/workflow_resume_detach_test.go | 12 +-- .../controlapi/workflow_resume_test.go | 36 ++++----- .../internal/controlapi/workflow_revert.go | 2 +- .../controlapi/workflow_revert_test.go | 28 +++---- .../internal/controlapi/workflow_suspend.go | 10 +-- .../controlapi/workflow_suspend_async_test.go | 6 +- .../controlapi/workflow_suspend_test.go | 80 +++++++++---------- .../internal/controlapi/workflow_tag.go | 10 +-- .../internal/controlapi/workflow_tag_test.go | 28 +++---- .../internal/controlapi/workflow_test.go | 2 +- .../controlapi/workflow_testutil_test.go | 4 +- .../controlapi/workflow_worker_delete_test.go | 6 +- cmd/ateapi/internal/store/atepg/tag.go | 6 +- cmd/ateapi/internal/store/atepg/tag_test.go | 2 +- .../internal/store/storecontract/contract.go | 18 ++--- cmd/ateapi/internal/workerservice/jwt_test.go | 4 +- cmd/kubectl-ate/internal/cmd/tag_test.go | 2 +- cmd/kubectl-ate/internal/printer/printer.go | 2 +- .../internal/printer/printer_test.go | 6 +- internal/e2e/suites/demo/demo_test.go | 2 +- .../e2e/suites/security/crossatespace_test.go | 2 +- .../snapshotcredentials_test.go | 2 +- .../actorscope/actorscope.go | 2 +- .../actorscope/actorscope_test.go | 8 +- .../exchange/exchange_test.go | 2 +- 49 files changed, 334 insertions(+), 334 deletions(-) diff --git a/api/ateapi/v1/ateapi.pb.go b/api/ateapi/v1/ateapi.pb.go index 250a2f5968..8b0cf6da99 100644 --- a/api/ateapi/v1/ateapi.pb.go +++ b/api/ateapi/v1/ateapi.pb.go @@ -89,55 +89,55 @@ func (SnapshotOwner) EnumDescriptor() ([]byte, []int) { return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{0} } -// SnapshotStorageStatus indicates the lifecycle stage of a durable snapshot. -type SnapshotStorageStatus int32 +// SnapshotStorageState indicates the lifecycle stage of a durable snapshot. +type SnapshotStorageState int32 const ( - SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED SnapshotStorageStatus = 0 + SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED SnapshotStorageState = 0 // Snapshot creation in durable storage is in progress. - SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING SnapshotStorageStatus = 1 + SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING SnapshotStorageState = 1 // Snapshot is complete and ready for use in durable storage. - SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED SnapshotStorageStatus = 2 // Keep this in sync with SnapshotStorage.status's maximum. + SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED SnapshotStorageState = 2 // Keep this in sync with SnapshotStorage.state's maximum. ) -// Enum value maps for SnapshotStorageStatus. +// Enum value maps for SnapshotStorageState. var ( - SnapshotStorageStatus_name = map[int32]string{ - 0: "SNAPSHOT_STORAGE_STATUS_UNSPECIFIED", - 1: "SNAPSHOT_STORAGE_STATUS_PENDING", - 2: "SNAPSHOT_STORAGE_STATUS_COMPLETED", + SnapshotStorageState_name = map[int32]string{ + 0: "SNAPSHOT_STORAGE_STATE_UNSPECIFIED", + 1: "SNAPSHOT_STORAGE_STATE_PENDING", + 2: "SNAPSHOT_STORAGE_STATE_COMPLETED", } - SnapshotStorageStatus_value = map[string]int32{ - "SNAPSHOT_STORAGE_STATUS_UNSPECIFIED": 0, - "SNAPSHOT_STORAGE_STATUS_PENDING": 1, - "SNAPSHOT_STORAGE_STATUS_COMPLETED": 2, + SnapshotStorageState_value = map[string]int32{ + "SNAPSHOT_STORAGE_STATE_UNSPECIFIED": 0, + "SNAPSHOT_STORAGE_STATE_PENDING": 1, + "SNAPSHOT_STORAGE_STATE_COMPLETED": 2, } ) -func (x SnapshotStorageStatus) Enum() *SnapshotStorageStatus { - p := new(SnapshotStorageStatus) +func (x SnapshotStorageState) Enum() *SnapshotStorageState { + p := new(SnapshotStorageState) *p = x return p } -func (x SnapshotStorageStatus) String() string { +func (x SnapshotStorageState) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } -func (SnapshotStorageStatus) Descriptor() protoreflect.EnumDescriptor { +func (SnapshotStorageState) Descriptor() protoreflect.EnumDescriptor { return file_api_ateapi_v1_ateapi_proto_enumTypes[1].Descriptor() } -func (SnapshotStorageStatus) Type() protoreflect.EnumType { +func (SnapshotStorageState) Type() protoreflect.EnumType { return &file_api_ateapi_v1_ateapi_proto_enumTypes[1] } -func (x SnapshotStorageStatus) Number() protoreflect.EnumNumber { +func (x SnapshotStorageState) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } -// Deprecated: Use SnapshotStorageStatus.Descriptor instead. -func (SnapshotStorageStatus) EnumDescriptor() ([]byte, []int) { +// Deprecated: Use SnapshotStorageState.Descriptor instead. +func (SnapshotStorageState) EnumDescriptor() ([]byte, []int) { return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{1} } @@ -640,12 +640,12 @@ func (x *ObjectSnapshot) GetSnapshotUri() string { // SnapshotStorage records the durable storage representation of a Snapshot. type SnapshotStorage struct { state protoimpl.MessageState `protogen:"open.v1"` - // status is the lifecycle status of the durable snapshot. + // state is the lifecycle state of the durable snapshot. // // +k8s:required // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotStorageStatus enum - Status SnapshotStorageStatus `protobuf:"varint,1,opt,name=status,proto3,enum=substrate.ateapi.v1.SnapshotStorageStatus" json:"status,omitempty"` + // +k8s:maximum=2 # keep this in sync with the SnapshotStorageState enum + State SnapshotStorageState `protobuf:"varint,1,opt,name=state,proto3,enum=substrate.ateapi.v1.SnapshotStorageState" json:"state,omitempty"` // object snapshot is stored in object storage. // // +k8s:required @@ -684,11 +684,11 @@ func (*SnapshotStorage) Descriptor() ([]byte, []int) { return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{1} } -func (x *SnapshotStorage) GetStatus() SnapshotStorageStatus { +func (x *SnapshotStorage) GetState() SnapshotStorageState { if x != nil { - return x.Status + return x.State } - return SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED + return SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED } func (x *SnapshotStorage) GetObject() *ObjectSnapshot { @@ -10031,9 +10031,9 @@ const file_api_ateapi_v1_ateapi_proto_rawDesc = "" + "\n" + "\x1aapi/ateapi/v1/ateapi.proto\x12\x13substrate.ateapi.v1\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"3\n" + "\x0eObjectSnapshot\x12!\n" + - "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"\x92\x01\n" + - "\x0fSnapshotStorage\x12B\n" + - "\x06status\x18\x01 \x01(\x0e2*.substrate.ateapi.v1.SnapshotStorageStatusR\x06status\x12;\n" + + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"\x8f\x01\n" + + "\x0fSnapshotStorage\x12?\n" + + "\x05state\x18\x01 \x01(\x0e2).substrate.ateapi.v1.SnapshotStorageStateR\x05state\x12;\n" + "\x06object\x18\x02 \x01(\v2#.substrate.ateapi.v1.ObjectSnapshotR\x06object\"\xa9\x03\n" + "\bSnapshot\x12\x12\n" + "\x04uuid\x18\x01 \x01(\tR\x04uuid\x12\x1e\n" + @@ -10533,11 +10533,11 @@ const file_api_ateapi_v1_ateapi_proto_rawDesc = "" + "\rSnapshotOwner\x12\x1e\n" + "\x1aSNAPSHOT_OWNER_UNSPECIFIED\x10\x00\x12\x18\n" + "\x14SNAPSHOT_OWNER_ACTOR\x10\x01\x12\x16\n" + - "\x12SNAPSHOT_OWNER_TAG\x10\x02*\x8c\x01\n" + - "\x15SnapshotStorageStatus\x12'\n" + - "#SNAPSHOT_STORAGE_STATUS_UNSPECIFIED\x10\x00\x12#\n" + - "\x1fSNAPSHOT_STORAGE_STATUS_PENDING\x10\x01\x12%\n" + - "!SNAPSHOT_STORAGE_STATUS_COMPLETED\x10\x02*\x90\x01\n" + + "\x12SNAPSHOT_OWNER_TAG\x10\x02*\x88\x01\n" + + "\x14SnapshotStorageState\x12&\n" + + "\"SNAPSHOT_STORAGE_STATE_UNSPECIFIED\x10\x00\x12\"\n" + + "\x1eSNAPSHOT_STORAGE_STATE_PENDING\x10\x01\x12$\n" + + " SNAPSHOT_STORAGE_STATE_COMPLETED\x10\x02*\x90\x01\n" + "\x10SnapshotFidelity\x12!\n" + "\x1dSNAPSHOT_FIDELITY_UNSPECIFIED\x10\x00\x12\x1d\n" + "\x19SNAPSHOT_FIDELITY_VOLUMES\x10\x01\x12\x1c\n" + @@ -10699,7 +10699,7 @@ var file_api_ateapi_v1_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 10) var file_api_ateapi_v1_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 144) var file_api_ateapi_v1_ateapi_proto_goTypes = []any{ (SnapshotOwner)(0), // 0: substrate.ateapi.v1.SnapshotOwner - (SnapshotStorageStatus)(0), // 1: substrate.ateapi.v1.SnapshotStorageStatus + (SnapshotStorageState)(0), // 1: substrate.ateapi.v1.SnapshotStorageState (SnapshotFidelity)(0), // 2: substrate.ateapi.v1.SnapshotFidelity (TagScope)(0), // 3: substrate.ateapi.v1.TagScope (ActorState)(0), // 4: substrate.ateapi.v1.ActorState @@ -10857,7 +10857,7 @@ var file_api_ateapi_v1_ateapi_proto_goTypes = []any{ (*descriptorpb.MethodOptions)(nil), // 156: google.protobuf.MethodOptions } var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ - 1, // 0: substrate.ateapi.v1.SnapshotStorage.status:type_name -> substrate.ateapi.v1.SnapshotStorageStatus + 1, // 0: substrate.ateapi.v1.SnapshotStorage.state:type_name -> substrate.ateapi.v1.SnapshotStorageState 10, // 1: substrate.ateapi.v1.SnapshotStorage.object:type_name -> substrate.ateapi.v1.ObjectSnapshot 0, // 2: substrate.ateapi.v1.Snapshot.owner:type_name -> substrate.ateapi.v1.SnapshotOwner 2, // 3: substrate.ateapi.v1.Snapshot.fidelity:type_name -> substrate.ateapi.v1.SnapshotFidelity diff --git a/api/ateapi/v1/ateapi.proto b/api/ateapi/v1/ateapi.proto index a5e777a4a6..d2b365c2b3 100644 --- a/api/ateapi/v1/ateapi.proto +++ b/api/ateapi/v1/ateapi.proto @@ -291,24 +291,24 @@ enum SnapshotOwner { // Keep this in sync with Snapshot.owner's maximum. } -// SnapshotStorageStatus indicates the lifecycle stage of a durable snapshot. -enum SnapshotStorageStatus { - SNAPSHOT_STORAGE_STATUS_UNSPECIFIED = 0; +// SnapshotStorageState indicates the lifecycle stage of a durable snapshot. +enum SnapshotStorageState { + SNAPSHOT_STORAGE_STATE_UNSPECIFIED = 0; // Snapshot creation in durable storage is in progress. - SNAPSHOT_STORAGE_STATUS_PENDING = 1; + SNAPSHOT_STORAGE_STATE_PENDING = 1; // Snapshot is complete and ready for use in durable storage. - SNAPSHOT_STORAGE_STATUS_COMPLETED = 2; - // Keep this in sync with SnapshotStorage.status's maximum. + SNAPSHOT_STORAGE_STATE_COMPLETED = 2; + // Keep this in sync with SnapshotStorage.state's maximum. } // SnapshotStorage records the durable storage representation of a Snapshot. message SnapshotStorage { - // status is the lifecycle status of the durable snapshot. + // state is the lifecycle state of the durable snapshot. // // +k8s:required // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotStorageStatus enum - SnapshotStorageStatus status = 1; + // +k8s:maximum=2 # keep this in sync with the SnapshotStorageState enum + SnapshotStorageState state = 1; // object snapshot is stored in object storage. // diff --git a/cmd/ateapi/internal/apivalidation/validation_test.go b/cmd/ateapi/internal/apivalidation/validation_test.go index 43b315af58..768d7e1cde 100644 --- a/cmd/ateapi/internal/apivalidation/validation_test.go +++ b/cmd/ateapi/internal/apivalidation/validation_test.go @@ -1068,7 +1068,7 @@ func validObjectSnapshot(mutate ...func(*ateapi.ObjectSnapshot)) *ateapi.ObjectS func validSnapshotStorage(mutate ...func(*ateapi.SnapshotStorage)) *ateapi.SnapshotStorage { ss := &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: validObjectSnapshot(), } for _, m := range mutate { @@ -1098,7 +1098,7 @@ func badSnapshot(mutate ...func(*ateapi.Snapshot)) *ateapi.Snapshot { breakIt := func(s *ateapi.Snapshot) { s.Fidelity = ateapi.SnapshotFidelity(4) s.DurableSnapshot = &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: ""}, } } @@ -1113,7 +1113,7 @@ func TestValidateSnapshot(t *testing.T) { fidelityPath := field.NewPath("fidelity") localityPath := field.NewPath("locality") durableSnapshotPath := field.NewPath("durable_snapshot") - statusPath := durableSnapshotPath.Child("status") + statePath := durableSnapshotPath.Child("state") objectPath := durableSnapshotPath.Child("object") uriPath := objectPath.Child("snapshot_uri") @@ -1185,25 +1185,25 @@ func TestValidateSnapshot(t *testing.T) { obj: valid(func(s *ateapi.Snapshot) { s.DurableSnapshot = nil }), }, { - name: "missing durable_snapshot.status", + name: "missing durable_snapshot.state", obj: valid(func(s *ateapi.Snapshot) { - s.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED + s.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED }), - want: field.ErrorList{field.Required(statusPath, "")}, + want: field.ErrorList{field.Required(statePath, "")}, }, { - name: "negative durable_snapshot.status", + name: "negative durable_snapshot.state", obj: valid(func(s *ateapi.Snapshot) { - s.DurableSnapshot.Status = ateapi.SnapshotStorageStatus(-1) + s.DurableSnapshot.State = ateapi.SnapshotStorageState(-1) }), - want: field.ErrorList{field.Invalid(statusPath, nil, "").WithOrigin("minimum")}, + want: field.ErrorList{field.Invalid(statePath, nil, "").WithOrigin("minimum")}, }, { - name: "durable_snapshot.status above the enum", + name: "durable_snapshot.state above the enum", obj: valid(func(s *ateapi.Snapshot) { - s.DurableSnapshot.Status = ateapi.SnapshotStorageStatus(3) + s.DurableSnapshot.State = ateapi.SnapshotStorageState(3) }), - want: field.ErrorList{field.Invalid(statusPath, nil, "").WithOrigin("maximum")}, + want: field.ErrorList{field.Invalid(statePath, nil, "").WithOrigin("maximum")}, }, { name: "missing durable_snapshot.object", diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index 5d32a05e7c..26cca003a0 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -8499,10 +8499,10 @@ func Validate_SnapshotStorage( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *v1.SnapshotStorage) (errs field.ErrorList) { - { // field v1.SnapshotStorage.Status + { // field v1.SnapshotStorage.State fn := func( fldPath *field.Path, - obj, oldObj *v1.SnapshotStorageStatus, + obj, oldObj *v1.SnapshotStorageState, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -8528,10 +8528,10 @@ func Validate_SnapshotStorage( return } oldVal := safe.Field(oldObj, - func(oldObj *v1.SnapshotStorage) *v1.SnapshotStorageStatus { - return &oldObj.Status + func(oldObj *v1.SnapshotStorage) *v1.SnapshotStorageState { + return &oldObj.State }) - errs = append(errs, fn(fldPath.Child("status"), &obj.Status, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("state"), &obj.State, oldVal, oldObj != nil)...) } { // field v1.SnapshotStorage.Object diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index ffee8080ea..0e56a6ae70 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -194,7 +194,7 @@ func (s *ServiceImpl) resolveTagSource(ctx context.Context, actorAtespace string // TODO: Refuse a tag with delete_time set. DeleteTag may already have // collected its snapshot. // A tag might have an empty Snapshot URI if the tag creation failed or is ongoing. - if tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { + if tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) == "" { return nil, apierror.FailedPrecondition("source Tag is still being created or failed creation") } // TODO: Permit compatible VOLUMES snapshots when runtimes can extract portable data. diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index b68f8ff73b..5ff334cbd9 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -259,7 +259,7 @@ func TestDeleteActorTemplate(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), actorURI.Name(), actorURI.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), actorURI.Name(), actorURI.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } }) var tag *ateapi.Tag diff --git a/cmd/ateapi/internal/controlapi/actor_test.go b/cmd/ateapi/internal/controlapi/actor_test.go index 29b4057422..f1c3b5fd96 100644 --- a/cmd/ateapi/internal/controlapi/actor_test.go +++ b/cmd/ateapi/internal/controlapi/actor_test.go @@ -405,7 +405,7 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { snap := newPausedSnapshot(t, 1, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "pause-snap", "node-1") if name == "uploaded-paused-actor" { snap.DurableSnapshot = &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: uploaded.String()}, } } @@ -615,9 +615,9 @@ func TestUpdateActor_RepointTemplateStorageLocation(t *testing.T) { }) if tt.snapshot != nil { uri := tt.snapshot(t, actor) - storageStatus := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + storageState := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED if tt.pending { - storageStatus = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + storageState = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING } actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 @@ -625,7 +625,7 @@ func TestUpdateActor_RepointTemplateStorageLocation(t *testing.T) { s.LastAssignedGeneration = tt.lastAssignedGen } s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot", uri, storageStatus), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot", uri, storageState), } }) } @@ -990,7 +990,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { Scope: ateapi.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapi.TagStatus{ ActorTemplateUid: tmpl.GetMetadata().GetUid(), - Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), someActorUID, "gs://bucket/atespaces/team-a/tags/"+someActorUID, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), someActorUID, "gs://bucket/atespaces/team-a/tags/"+someActorUID, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), }, } wantCode := codes.OK @@ -1021,7 +1021,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { actor := &ateapi.Actor{Metadata: &ateapi.ResourceMetadata{Atespace: "team-a", Name: "actor"}, ActorTemplate: resources.ActorTemplateRefFromActorTemplate(tmpl).ToObjectRef()} if scenario == "explicit tag" { tag.Metadata.Name = "explicit" - tag.Status.Snapshot = newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "explicit", "gs://bucket/atespaces/team-a/tags/explicit", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + tag.Status.Snapshot = newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "explicit", "gs://bucket/atespaces/team-a/tags/explicit", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if _, err := persistence.CreateTag(ctx, tag); err != nil { t.Fatal(err) } @@ -1035,8 +1035,8 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { if err != nil { return } - gotSnap := findLatestDurableSnapshot(created.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) - if gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) || gotSnap.GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { + gotSnap := findLatestDurableSnapshot(created.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) + if gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) || gotSnap.GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { t.Fatalf("incorrect initial status: %v", created.GetStatus()) } if _, err := persistence.GetEgressPolicy(ctx, resources.ActorRefFromActor(created)); !errors.Is(err, store.ErrNotFound) { @@ -1050,7 +1050,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { if _, err := persistence.UpdateActor(ctx, resources.ActorRefFromActor(created), store.PreconditionFrom(created), func(db *ateapi.Actor) error { db.Status.LastAssignedGeneration = 1 db.Status.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } return nil }); err != nil { diff --git a/cmd/ateapi/internal/controlapi/crash_test.go b/cmd/ateapi/internal/controlapi/crash_test.go index 03d6d925c6..2e9e584585 100644 --- a/cmd/ateapi/internal/controlapi/crash_test.go +++ b/cmd/ateapi/internal/controlapi/crash_test.go @@ -65,7 +65,7 @@ func seedActor(t *testing.T, ctx context.Context, st store.Interface, actorRef r }, LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "reserved-snapshot", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "reserved-snapshot", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, }) @@ -112,7 +112,7 @@ func seedUnboundActor(t *testing.T, ctx context.Context, st store.Interface, act State: ateapi.ActorState_ACTOR_STATE_RUNNING, LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "reserved-snapshot", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "reserved-snapshot", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, }) @@ -130,7 +130,7 @@ func assertCrashed(t *testing.T, ctx context.Context, st store.Interface, actorR t.Errorf("status = %v, want %v", got.GetStatus().GetState(), ateapi.ActorState_ACTOR_STATE_CRASHED) } // Keep the snapshot uri for debugging. - if findLatestDurableSnapshot(got.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) == nil { + if findLatestDurableSnapshot(got.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) == nil { t.Error(`in-progress durable snapshot URI = "", want preserved`) } if got.GetStatus().GetWorkerAssignment() != nil { diff --git a/cmd/ateapi/internal/controlapi/export_test.go b/cmd/ateapi/internal/controlapi/export_test.go index c1953553a8..81f1d80aec 100644 --- a/cmd/ateapi/internal/controlapi/export_test.go +++ b/cmd/ateapi/internal/controlapi/export_test.go @@ -97,7 +97,7 @@ func ReleaseWindowWorkflows() []ReleaseWindowWorkflow { Name: "suspend", State: ateapi.ActorState_ACTOR_STATE_SUSPENDING, Snapshot: func(t *testing.T) *ateapi.Snapshot { - return newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + return newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) }, // As for the async suspend, the finalize commit is the first actor update. Finalize: func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapi.Actor) error { diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 7544c3bd2a..0934da4e22 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -448,7 +448,7 @@ func TestCreateActor_PendingTag(t *testing.T) { // Simulates a tag creation that failed while writing the snapshot to // external storage. pending := seedTag(t, tc, "pending-source", "pending", func(tag *ateapi.Tag) { - tag.Status.Snapshot.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + tag.Status.Snapshot.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING tag.Status.ActorTemplateUid = tmpl.GetMetadata().GetUid() }) tagRef := &ateapi.ObjectRef{Atespace: testAtespace, Name: "pending"} @@ -466,7 +466,7 @@ func TestCreateActor_PendingTag(t *testing.T) { if _, err := tc.persistence.UpdateTag(ctx, resources.TagRefFromTag(pending), store.PreconditionFrom(pending), func(toUpdate *ateapi.Tag) error { - toUpdate.Status.Snapshot.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + toUpdate.Status.Snapshot.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED return nil }); err != nil { t.Fatalf("finalizing the tag: %v", err) @@ -1197,7 +1197,7 @@ func TestDeleteActor_MissingSnapshotBucket(t *testing.T) { if state != ateapi.ActorState_ACTOR_STATE_SUSPENDED { // A failed upload leaves only the in-progress URI, even // when a prior deletion already marked the actor DELETING. - snap.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + snap.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING } toUpdate.Status.Snapshots = []*ateapi.Snapshot{snap} return nil @@ -4212,8 +4212,8 @@ func TestResumeActor_SyncSuspendKeepsLocalCopy(t *testing.T) { if snap == nil || snap.GetLocality() != "node1" { t.Fatalf("sync snapshot = %v, want its local copy on node1 recorded", snap) } - if got := snap.GetDurableSnapshot().GetStatus(); got != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { - t.Fatalf("sync snapshot durable status = %v, want COMPLETED", got) + if got := snap.GetDurableSnapshot().GetState(); got != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { + t.Fatalf("sync snapshot durable state = %v, want COMPLETED", got) } if got := snap.GetActorTemplateUid(); got != tmpl.GetMetadata().GetUid() { t.Fatalf("snapshot template UID = %q, want original template %q", got, tmpl.GetMetadata().GetUid()) @@ -4712,8 +4712,8 @@ func TestResumeActor_PausedPendingSnapshotRefusesRuntimeFallback(t *testing.T) { if paused == nil { t.Fatalf("pause left no local snapshot: %v", pauseResp.GetActor().GetStatus()) } - if got := paused.GetDurableSnapshot().GetStatus(); got != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { - t.Fatalf("pause checkpoint durable status = %v, want PENDING (not yet uploaded)", got) + if got := paused.GetDurableSnapshot().GetState(); got != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING { + t.Fatalf("pause checkpoint durable state = %v, want PENDING (not yet uploaded)", got) } durable := durableSnapshot(pauseResp.GetActor().GetStatus()) if durable == nil || durable.GetGeneration() >= paused.GetGeneration() { @@ -4750,7 +4750,7 @@ func TestResumeActor_PausedPendingSnapshotRefusesRuntimeFallback(t *testing.T) { } stillLocal := localSnapshot(after.GetStatus()) if stillLocal == nil || stillLocal.GetUuid() != paused.GetUuid() || - stillLocal.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { + stillLocal.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING { t.Errorf("pause checkpoint after the refused resume = %v, want %v still PENDING", stillLocal, paused) } if got := durableSnapshotURI(after.GetStatus()); got != durable.GetDurableSnapshot().GetObject().GetSnapshotUri() { @@ -5795,7 +5795,7 @@ func TestSuspendActor_SnapshotConflictPreservesState(t *testing.T) { pending = snap } } - if pending.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING || pending.GetLocality() != "" { + if pending.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING || pending.GetLocality() != "" { t.Fatalf("in-progress snapshot = %v, want PENDING without locality", pending) } assertSnapshotCollected(t, tc, pending.GetDurableSnapshot().GetObject().GetSnapshotUri()) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index d4cdfd5230..d3e0937b97 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -527,7 +527,7 @@ func newDurableSnapshot(gen int32, owner ateapi.SnapshotOwner, fidelity ateapi.S Fidelity: fidelity, ActorTemplateUid: templateUID, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: uri}, }, } @@ -550,7 +550,7 @@ func newPausedSnapshot(t *testing.T, gen int32, fidelity ateapi.SnapshotFidelity ActorTemplateUid: templateUID, Locality: locality, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING, Object: &ateapi.ObjectSnapshot{SnapshotUri: uri.String()}, }, } @@ -560,7 +560,7 @@ func durableSnapshot(status *ateapi.ActorStatus) *ateapi.Snapshot { var best *ateapi.Snapshot for _, snap := range status.GetSnapshots() { st := snap.GetDurableSnapshot() - if st.GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED && + if st.GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED && st.GetObject().GetSnapshotUri() != "" { if best == nil || snap.GetGeneration() > best.GetGeneration() { best = snap diff --git a/cmd/ateapi/internal/controlapi/hosting_test.go b/cmd/ateapi/internal/controlapi/hosting_test.go index e32474d057..3537fae1c1 100644 --- a/cmd/ateapi/internal/controlapi/hosting_test.go +++ b/cmd/ateapi/internal/controlapi/hosting_test.go @@ -235,8 +235,8 @@ func TestHostingNodeAfterResumeOntoNewWorker(t *testing.T) { // TestHostingNodeWithoutWorker pins which no-worker shapes count assigned_node // as hosting the actor. None of them reads the store. func TestHostingNodeWithoutWorker(t *testing.T) { - durable := func(st ateapi.SnapshotStorageStatus) *ateapi.SnapshotStorage { - return &ateapi.SnapshotStorage{Status: st, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://snaps/root/atespaces/a/actors/uid/snap-1"}} + durable := func(st ateapi.SnapshotStorageState) *ateapi.SnapshotStorage { + return &ateapi.SnapshotStorage{State: st, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://snaps/root/atespaces/a/actors/uid/snap-1"}} } status := func(state ateapi.ActorState, locality string, d *ateapi.SnapshotStorage) *ateapi.ActorStatus { return &ateapi.ActorStatus{ @@ -246,9 +246,9 @@ func TestHostingNodeWithoutWorker(t *testing.T) { Snapshots: []*ateapi.Snapshot{{Uuid: "snap-1", Generation: 1, Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, Locality: locality, DurableSnapshot: d}}, } } - olderOnNode := status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + olderOnNode := status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)) olderOnNode.LastAssignedGeneration = 2 - olderOnNode.Snapshots = append(olderOnNode.Snapshots, &ateapi.Snapshot{Uuid: "snap-2", Generation: 2, Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, DurableSnapshot: durable(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)}) + olderOnNode.Snapshots = append(olderOnNode.Snapshots, &ateapi.Snapshot{Uuid: "snap-2", Generation: 2, Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, DurableSnapshot: durable(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)}) noAssignedNode := status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", nil) noAssignedNode.AssignedNode = "" for _, tc := range []struct { @@ -257,11 +257,11 @@ func TestHostingNodeWithoutWorker(t *testing.T) { want string }{ {"paused", status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", nil), "node-1"}, - {"paused, upload pending", status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)), "node-1"}, - {"paused, uploaded, local copy kept", status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)), "node-1"}, + {"paused, upload pending", status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)), "node-1"}, + {"paused, uploaded, local copy kept", status(ateapi.ActorState_ACTOR_STATE_SUSPENDED, "node-1", durable(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)), "node-1"}, {"older snapshot on the node, latest not", olderOnNode, "node-1"}, {"local copy but no assigned node", noAssignedNode, ""}, - {"suspending without a worker", status(ateapi.ActorState_ACTOR_STATE_SUSPENDING, "node-1", durable(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)), ""}, + {"suspending without a worker", status(ateapi.ActorState_ACTOR_STATE_SUSPENDING, "node-1", durable(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)), ""}, {"crashed with a local copy", status(ateapi.ActorState_ACTOR_STATE_CRASHED, "node-1", nil), ""}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers.go b/cmd/ateapi/internal/controlapi/snapshot_helpers.go index 3e29b78312..1735536310 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers.go @@ -53,10 +53,10 @@ func latestSnapshot(status *ateapi.ActorStatus, matches func(*ateapi.Snapshot) b } // findLatestDurableSnapshot returns the highest-generation Snapshot on status -// whose DurableSnapshot matches storageStatus, or nil if none exists. -func findLatestDurableSnapshot(status *ateapi.ActorStatus, storageStatus ateapi.SnapshotStorageStatus) *ateapi.Snapshot { +// whose DurableSnapshot matches storageState, or nil if none exists. +func findLatestDurableSnapshot(status *ateapi.ActorStatus, storageState ateapi.SnapshotStorageState) *ateapi.Snapshot { return latestSnapshot(status, func(snap *ateapi.Snapshot) bool { - return snap.GetDurableSnapshot().GetStatus() == storageStatus + return snap.GetDurableSnapshot().GetState() == storageState }) } @@ -107,7 +107,7 @@ func pruneSnapshots(status *ateapi.ActorStatus, local *ateapi.Snapshot) { if status == nil { return } - latestDurable := findLatestDurableSnapshot(status, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + latestDurable := findLatestDurableSnapshot(status, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) kept := status.Snapshots[:0] for _, snap := range status.Snapshots { if snap != local { @@ -121,7 +121,7 @@ func pruneSnapshots(status *ateapi.ActorStatus, local *ateapi.Snapshot) { } // newDurableSnapshot constructs a Snapshot with DurableSnapshot populated. -func newDurableSnapshot(gen int32, owner ateapi.SnapshotOwner, fidelity ateapi.SnapshotFidelity, templateUID, uuid, uri string, storageStatus ateapi.SnapshotStorageStatus) *ateapi.Snapshot { +func newDurableSnapshot(gen int32, owner ateapi.SnapshotOwner, fidelity ateapi.SnapshotFidelity, templateUID, uuid, uri string, storageState ateapi.SnapshotStorageState) *ateapi.Snapshot { return &ateapi.Snapshot{ Uuid: uuid, Generation: gen, @@ -129,18 +129,18 @@ func newDurableSnapshot(gen int32, owner ateapi.SnapshotOwner, fidelity ateapi.S Fidelity: fidelity, ActorTemplateUid: templateUID, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: storageStatus, + State: storageState, Object: &ateapi.ObjectSnapshot{SnapshotUri: uri}, }, } } // tagDurableSnapshotURI returns the durable snapshot URI on tag matching -// storageStatus (or any status if storageStatus is UNSPECIFIED), or "" if none +// storageState (or any state if storageState is UNSPECIFIED), or "" if none // exists. -func tagDurableSnapshotURI(tag *ateapi.Tag, storageStatus ateapi.SnapshotStorageStatus) string { +func tagDurableSnapshotURI(tag *ateapi.Tag, storageState ateapi.SnapshotStorageState) string { st := tag.GetStatus().GetSnapshot().GetDurableSnapshot() - if storageStatus != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED && st.GetStatus() != storageStatus { + if storageState != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED && st.GetState() != storageState { return "" } return st.GetObject().GetSnapshotUri() diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go index 760cb85a3f..791bcbeccb 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go @@ -34,7 +34,7 @@ func TestFindSnapshotByGenerationAndSnapshotAtLatestGeneration(t *testing.T) { } s1 := newPausedSnapshot(t, 1, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-1", "node-1") - s2 := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + s2 := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) status := &ateapi.ActorStatus{ LastAssignedGeneration: 2, Snapshots: []*ateapi.Snapshot{s1, s2}, @@ -52,16 +52,16 @@ func TestFindSnapshotByGenerationAndSnapshotAtLatestGeneration(t *testing.T) { } func TestFindLatestDurableAndLocalSnapshots(t *testing.T) { - if snap := findLatestDurableSnapshot(nil, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); snap != nil { + if snap := findLatestDurableSnapshot(nil, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED); snap != nil { t.Errorf("findLatestDurableSnapshot(nil) = %v, want nil", snap) } if snap := findLatestLocalSnapshot(nil); snap != nil { t.Errorf("findLatestLocalSnapshot(nil) = %v, want nil", snap) } - s1 := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) - s2 := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) - s3 := newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + s1 := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) + s2 := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) + s3 := newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) s4 := newPausedSnapshot(t, 4, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-4", "node-1") // Place s2 before s1 to verify generation comparison rather than slice order. @@ -69,17 +69,17 @@ func TestFindLatestDurableAndLocalSnapshots(t *testing.T) { Snapshots: []*ateapi.Snapshot{s2, s1, s3, s4}, } - gotSnap := findLatestDurableSnapshot(status, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(status, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if gotSnap != s2 || gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != "gs://b/snap-2" { t.Errorf("findLatestDurableSnapshot(COMPLETED) = %v, want gen 2 (gs://b/snap-2)", gotSnap) } // An async suspend snapshot's durable copy is PENDING until atelet uploads it. - gotSnap = findLatestDurableSnapshot(status, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + gotSnap = findLatestDurableSnapshot(status, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) if gotSnap != s4 { t.Errorf("findLatestDurableSnapshot(PENDING) = %v, want gen 4", gotSnap) } - gotSnap = findLatestDurableSnapshot(&ateapi.ActorStatus{Snapshots: []*ateapi.Snapshot{s2, s1, s3}}, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + gotSnap = findLatestDurableSnapshot(&ateapi.ActorStatus{Snapshots: []*ateapi.Snapshot{s2, s1, s3}}, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) if gotSnap != s3 || gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != "gs://b/snap-3" { t.Errorf("findLatestDurableSnapshot(PENDING) without the paused snapshot = %v, want gen 3 (gs://b/snap-3)", gotSnap) } @@ -114,7 +114,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { }) t.Run("pause flow: pruneSnapshots keeps the latest COMPLETED durable and the new local snapshot", func(t *testing.T) { - completedGen1 := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + completedGen1 := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) completedGen1.Locality = "node-1" localGen2 := newPausedSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-2") localGen3 := newPausedSnapshot(t, 3, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "") @@ -128,7 +128,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { pruneSnapshots(status, localGen3) want := []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), newPausedSnapshot(t, 3, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "node-3"), } if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { @@ -140,12 +140,12 @@ func TestSnapshotPruningHelpers(t *testing.T) { }) t.Run("pruneSnapshots drops a PENDING snapshot that still names a node", func(t *testing.T) { - pendingWithLocality := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + pendingWithLocality := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-2", "gs://b/snap-2", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) pendingWithLocality.Locality = "node-1" local := newPausedSnapshot(t, 3, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "node-1") status := &ateapi.ActorStatus{ Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), pendingWithLocality, local, }, @@ -154,7 +154,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { pruneSnapshots(status, local) want := []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), newPausedSnapshot(t, 3, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", "node-1"), } if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { @@ -166,20 +166,20 @@ func TestSnapshotPruningHelpers(t *testing.T) { }) t.Run("revert flow: pruneSnapshots drops all PENDING snapshots and retains latest COMPLETED", func(t *testing.T) { - completedWithLocality := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + completedWithLocality := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) completedWithLocality.Locality = "node-1" status := &ateapi.ActorStatus{ Snapshots: []*ateapi.Snapshot{ completedWithLocality, newPausedSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-2"), - newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, } pruneSnapshots(status, nil) want := []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) @@ -189,9 +189,9 @@ func TestSnapshotPruningHelpers(t *testing.T) { t.Run("suspend flow: pruneSnapshots drops older COMPLETED and PENDING snapshots", func(t *testing.T) { status := &ateapi.ActorStatus{ Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/snap-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), newPausedSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", "node-1"), - newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), }, } status.Snapshots[2].Locality = "node-1" @@ -199,7 +199,7 @@ func TestSnapshotPruningHelpers(t *testing.T) { pruneSnapshots(status, nil) want := []*ateapi.Snapshot{ - newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(3, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-3", "gs://b/snap-3", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) @@ -214,12 +214,12 @@ func TestLatestSnapshotIsLocal(t *testing.T) { local := func(gen int32) *ateapi.Snapshot { return newPausedSnapshot(t, gen, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "node-1") } - withCopy := func(st ateapi.SnapshotStorageStatus) *ateapi.Snapshot { + withCopy := func(st ateapi.SnapshotStorageState) *ateapi.Snapshot { snap := local(1) - snap.DurableSnapshot = &ateapi.SnapshotStorage{Status: st, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://b/up"}} + snap.DurableSnapshot = &ateapi.SnapshotStorage{State: st, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://b/up"}} return snap } - durable := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "durable", "gs://b/durable", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + durable := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "durable", "gs://b/durable", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) notCaptured := newPausedSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "") tests := []struct { @@ -230,8 +230,8 @@ func TestLatestSnapshotIsLocal(t *testing.T) { {"nil status", nil, false}, {"no snapshots", &ateapi.ActorStatus{}, false}, {"local only", &ateapi.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{local(1)}}, true}, - {"local and uploaded", &ateapi.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{withCopy(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)}}, true}, - {"local with an upload pending", &ateapi.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{withCopy(ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)}}, true}, + {"local and uploaded", &ateapi.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{withCopy(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)}}, true}, + {"local with an upload pending", &ateapi.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{withCopy(ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)}}, true}, {"durable only", &ateapi.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapi.Snapshot{durable}}, false}, {"older local under a pause in progress", &ateapi.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapi.Snapshot{local(1), notCaptured}}, false}, } @@ -245,22 +245,22 @@ func TestLatestSnapshotIsLocal(t *testing.T) { } func TestTagDurableSnapshotURI(t *testing.T) { - if got := tagDurableSnapshotURI(nil, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED); got != "" { + if got := tagDurableSnapshotURI(nil, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED); got != "" { t.Errorf("tagDurableSnapshotURI(nil) = %q, want empty", got) } pendingTag := &ateapi.Tag{ Status: &ateapi.TagStatus{ - Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/tag-1", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "uuid-1", "gs://b/tag-1", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, } - if got := tagDurableSnapshotURI(pendingTag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != "" { + if got := tagDurableSnapshotURI(pendingTag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED); got != "" { t.Errorf("tagDurableSnapshotURI(pendingTag, COMPLETED) = %q, want empty", got) } - if got := tagDurableSnapshotURI(pendingTag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING); got != "gs://b/tag-1" { + if got := tagDurableSnapshotURI(pendingTag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING); got != "gs://b/tag-1" { t.Errorf("tagDurableSnapshotURI(pendingTag, PENDING) = %q, want gs://b/tag-1", got) } - if got := tagDurableSnapshotURI(pendingTag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED); got != "gs://b/tag-1" { + if got := tagDurableSnapshotURI(pendingTag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED); got != "gs://b/tag-1" { t.Errorf("tagDurableSnapshotURI(pendingTag, UNSPECIFIED) = %q, want gs://b/tag-1", got) } } diff --git a/cmd/ateapi/internal/controlapi/snapshot_report.go b/cmd/ateapi/internal/controlapi/snapshot_report.go index b03b5ece68..e9c9693e9d 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_report.go +++ b/cmd/ateapi/internal/controlapi/snapshot_report.go @@ -165,7 +165,7 @@ func applyLatestDurable(ctx context.Context, status *ateapi.ActorStatus, reporte if reported == nil { return nil } - if latest := findLatestDurableSnapshot(status, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); latest.GetGeneration() >= reported.GetGeneration() { + if latest := findLatestDurableSnapshot(status, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED); latest.GetGeneration() >= reported.GetGeneration() { if latest.GetGeneration() > reported.GetGeneration() { slog.InfoContext(ctx, msgDurableOlder, slog.Int("generation", int(reported.GetGeneration())), @@ -208,7 +208,7 @@ func applyLatestDurable(ctx context.Context, status *ateapi.ActorStatus, reporte return nil } snap.Fidelity = reported.GetFidelity() - snap.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + snap.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED return nil } @@ -222,7 +222,7 @@ func applyLatestDurable(ctx context.Context, status *ateapi.ActorStatus, reporte func applyDetach(status *ateapi.ActorStatus, node string, upTo int32) error { deleted := func(s *ateapi.Snapshot) bool { return s.GetLocality() == node && s.GetGeneration() <= upTo } if latest := latestSnapshot(status, deleted); status.GetState() != ateapi.ActorState_ACTOR_STATE_CRASHED && latest != nil && - latest.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + latest.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { return apierror.FailedPrecondition("snapshot generation %d on node %s is not durable", latest.GetGeneration(), node) } for _, snap := range status.GetSnapshots() { diff --git a/cmd/ateapi/internal/controlapi/snapshot_report_test.go b/cmd/ateapi/internal/controlapi/snapshot_report_test.go index 9232068992..c3718f247d 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_report_test.go +++ b/cmd/ateapi/internal/controlapi/snapshot_report_test.go @@ -51,12 +51,12 @@ func ownedURI(t *testing.T, actorUID string, gen int32) string { } // withDurable returns a Snapshot held on locality ("" for none) with a durable -// copy in the given storage status. -func withDurable(t *testing.T, gen int32, locality string, storage ateapi.SnapshotStorageStatus) *ateapi.Snapshot { +// copy in the given storage state. +func withDurable(t *testing.T, gen int32, locality string, storage ateapi.SnapshotStorageState) *ateapi.Snapshot { t.Helper() snap := &ateapi.Snapshot{Uuid: snapName(gen), Generation: gen, Locality: locality, Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY} snap.DurableSnapshot = &ateapi.SnapshotStorage{ - Status: storage, + State: storage, Object: &ateapi.ObjectSnapshot{SnapshotUri: ownedURI(t, reportActorUID, gen)}, } return snap @@ -79,8 +79,8 @@ func reportedAt(t *testing.T, gen int32, uri string) *ateapi.SnapshotReport { func TestApplyActorSnapshotState(t *testing.T) { const ( - pending = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - completed = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pending = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING + completed = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED suspended = ateapi.ActorState_ACTOR_STATE_SUSPENDED running = ateapi.ActorState_ACTOR_STATE_RUNNING ) @@ -325,8 +325,8 @@ func TestApplyActorSnapshotState(t *testing.T) { // and one that is applied, aren't logged. func TestApplyLatestDurableLogsStaleReports(t *testing.T) { const ( - pending = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - completed = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pending = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING + completed = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED ) tests := []struct { name string @@ -433,7 +433,7 @@ func TestApplyReportedSnapshotState(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.AssignedNode = reportNode s.LastAssignedGeneration = 2 - snap := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", pending.Name(), pending.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + snap := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", pending.Name(), pending.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) snap.Locality = reportNode s.Snapshots = []*ateapi.Snapshot{snap} }) @@ -509,7 +509,7 @@ func TestApplyReportedSnapshotState(t *testing.T) { t.Run("applied to a "+st.String()+" actor", func(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.State = st - s.Snapshots[0].DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + s.Snapshots[0].DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING }) if err := w.ApplyReportedSnapshotState(ctx, actorRef, uid, reportNode, state); err != nil { t.Fatalf("ApplyReportedSnapshotState() failed: %v", err) @@ -518,7 +518,7 @@ func TestApplyReportedSnapshotState(t *testing.T) { if err != nil { t.Fatal(err) } - if st := got.GetStatus().GetSnapshots()[0].GetDurableSnapshot().GetStatus(); st != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + if st := got.GetStatus().GetSnapshots()[0].GetDurableSnapshot().GetState(); st != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { t.Errorf("durable snapshot status = %v, want COMPLETED", st) } @@ -533,7 +533,7 @@ func TestApplyReportedSnapshotState(t *testing.T) { // A suspend that crashed before it finalized leaves its snapshot on no node. older := mustActorSnapshotURI(t, tmpl, actor, "snap-1") - snapshot := func(gen int32, uri resources.SnapshotURI, locality string, storage ateapi.SnapshotStorageStatus) *ateapi.Snapshot { + snapshot := func(gen int32, uri resources.SnapshotURI, locality string, storage ateapi.SnapshotStorageState) *ateapi.Snapshot { snap := newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uri.Name(), uri.String(), storage) snap.Locality = locality return snap @@ -546,35 +546,35 @@ func TestApplyReportedSnapshotState(t *testing.T) { }{{ name: "a crashed suspend's upload stays pending", snapshots: []*ateapi.Snapshot{ - snapshot(1, older, reportNode, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - snapshot(2, pending, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + snapshot(1, older, reportNode, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + snapshot(2, pending, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, state: state, want: []*ateapi.Snapshot{ - snapshot(1, older, reportNode, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - snapshot(2, pending, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + snapshot(1, older, reportNode, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + snapshot(2, pending, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, { name: "a crashed suspend's detach releases the older snapshot", snapshots: []*ateapi.Snapshot{ - snapshot(1, older, reportNode, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - snapshot(2, pending, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + snapshot(1, older, reportNode, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + snapshot(2, pending, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, state: &ateapi.ActorSnapshotState{LatestDurable: uploaded}, want: []*ateapi.Snapshot{ - snapshot(1, older, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - snapshot(2, pending, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + snapshot(1, older, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + snapshot(2, pending, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, { name: "a crashed suspend's detach releases an older snapshot that isn't durable", snapshots: []*ateapi.Snapshot{ - snapshot(1, older, reportNode, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), - snapshot(2, pending, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + snapshot(1, older, reportNode, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), + snapshot(2, pending, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, state: &ateapi.ActorSnapshotState{LatestDurable: uploaded}, want: []*ateapi.Snapshot{ - snapshot(1, older, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), - snapshot(2, pending, "", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + snapshot(1, older, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), + snapshot(2, pending, "", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }} { t.Run(tc.name, func(t *testing.T) { @@ -601,7 +601,7 @@ func TestActorOnNode(t *testing.T) { actor := func(assigned string, localities ...string) *ateapi.Actor { status := &ateapi.ActorStatus{AssignedNode: assigned} for i, l := range localities { - status.Snapshots = append(status.Snapshots, withDurable(t, int32(i+1), l, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)) + status.Snapshots = append(status.Snapshots, withDurable(t, int32(i+1), l, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)) } return &ateapi.Actor{Status: status} } @@ -635,8 +635,8 @@ func ateletReported(t *testing.T, gen int32) *ateletpb.SnapshotReport { func TestApplyAteletResponseState(t *testing.T) { const ( - pending = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - completed = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pending = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING + completed = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED suspended = ateapi.ActorState_ACTOR_STATE_SUSPENDED suspending = ateapi.ActorState_ACTOR_STATE_SUSPENDING diff --git a/cmd/ateapi/internal/controlapi/tag.go b/cmd/ateapi/internal/controlapi/tag.go index ec9362c707..b53a44364b 100644 --- a/cmd/ateapi/internal/controlapi/tag.go +++ b/cmd/ateapi/internal/controlapi/tag.go @@ -123,7 +123,7 @@ func (s *RPCService) UpdateTag(ctx context.Context, req *ateapi.UpdateTagRequest // A tag whose create never finished names a partial copy. Publishing it // — or changing its scope at all — would hand out content that is still // being written, or may never be. - if tagDurableSnapshotURI(toUpdate, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { + if tagDurableSnapshotURI(toUpdate, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) == "" { return errTagPending } // Metadata and status are server-owned fields. diff --git a/cmd/ateapi/internal/controlapi/tag_test.go b/cmd/ateapi/internal/controlapi/tag_test.go index a79c3cc000..9bd244092a 100644 --- a/cmd/ateapi/internal/controlapi/tag_test.go +++ b/cmd/ateapi/internal/controlapi/tag_test.go @@ -135,7 +135,7 @@ func TestUpdateTag(t *testing.T) { req: &ateapi.Tag{ Scope: ateapi.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapi.TagStatus{ - Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, "", "elsewhere", "gs://attacker/elsewhere", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, "", "elsewhere", "gs://attacker/elsewhere", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), ActorTemplateUid: "other-template-uid", }, }, @@ -251,7 +251,7 @@ func newTestSuspendedActor(t *testing.T, ctx context.Context, st store.Interface return mustUpdateActorStatus(t, ctx, st, actor, func(status *ateapi.ActorStatus) { status.LastAssignedGeneration = 1 status.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uri.Name(), uri.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uri.Name(), uri.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } }) } @@ -266,13 +266,13 @@ func newTestTag(t *testing.T, name string, actor *ateapi.Actor) *ateapi.Tag { if err != nil { t.Fatalf("NewTagSnapshotURI: %v", err) } - actorSnap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + actorSnap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) return &ateapi.Tag{ Metadata: &ateapi.ResourceMetadata{Atespace: atespace, Name: name}, Scope: ateapi.TagScope_TAG_SCOPE_ATESPACE, SourceActor: resources.ActorRefFromActor(actor).ToObjectRef(), Status: &ateapi.TagStatus{ - Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, actorSnap.GetFidelity(), actorSnap.GetActorTemplateUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, actorSnap.GetFidelity(), actorSnap.GetActorTemplateUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), }, } } @@ -283,7 +283,7 @@ func newTestTag(t *testing.T, name string, actor *ateapi.Actor) *ateapi.Tag { func newPendingTestTag(t *testing.T, name string, actor *ateapi.Actor) *ateapi.Tag { t.Helper() tag := newTestTag(t, name, actor) - tag.Status.Snapshot.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + tag.Status.Snapshot.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING return tag } diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 285649afd2..810ca27682 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -236,7 +236,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource if tag.GetStatus().GetActorTemplateUid() != tmpl.GetMetadata().GetUid() || resources.ActorRefFromObjectRef(tag.GetSourceActor()) != resources.ActorRefFromObjectRef(goldenActorRef) { return 0, r.fail(ctx, tmpl, reasonGoldenTagConflict, "golden tag belongs to another actor or template") } - if tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) != "" { + if tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) != "" { return 0, r.saveGoldenTag(ctx, tmpl, goldenActorRef) } // CreateTag cannot resume an incomplete copy. Delete it before retrying. @@ -296,7 +296,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource ateapi.ActorState_ACTOR_STATE_SUSPENDED: // The golden actor was never resumed, or a previous resume didn't // finish; ResumeActor is reentrant from both. - if findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) != nil { + if findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) != nil { // Golden actors never start from a source snapshot, so an // existing snapshot means an earlier suspend completed // without being recorded. @@ -340,7 +340,7 @@ func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *a // A crash during suspend is observed as CRASHED on the retry. return fmt.Errorf("while suspending golden actor: %w", err) } - if findLatestDurableSnapshot(resp.GetActor().GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == nil { + if findLatestDurableSnapshot(resp.GetActor().GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) == nil { return fmt.Errorf("suspending golden actor produced no external snapshot") } return nil diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index 68c3ca2778..dfac9b56e5 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -188,7 +188,7 @@ func (c *fakeGoldenControl) goldenActorStatus() *ateapi.ActorStatus { if c.goldenSnapshot != "" { st.LastAssignedGeneration = 1 st.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "golden-snapshot", c.goldenSnapshot, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "golden-snapshot", c.goldenSnapshot, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } } return st @@ -283,7 +283,7 @@ func (c *fakeGoldenControl) CreateTag(_ context.Context, req *ateapi.CreateTagRe c.tag = proto.CloneOf(req.GetTag()) c.tag.Status = &ateapi.TagStatus{ ActorTemplateUid: testTemplateUID, - Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "golden-snapshot", c.goldenSnapshot, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "golden-snapshot", c.goldenSnapshot, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } return proto.CloneOf(c.tag), nil } @@ -1129,7 +1129,7 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { Scope: ateapi.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapi.TagStatus{ ActorTemplateUid: testTemplateUID, - Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "tag-snapshot", "gs://bucket/tag-snapshot", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + Snapshot: newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "tag-snapshot", "gs://bucket/tag-snapshot", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), }, } incomplete := proto.CloneOf(completed) @@ -1187,7 +1187,7 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { if !proto.Equal(st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus().GetGoldenTag(), ref) { t.Fatal("golden tag not recorded") } - if tagDurableSnapshotURI(control.tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { + if tagDurableSnapshotURI(control.tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) == "" { t.Fatal("golden tag has no snapshot") } if len(control.createReqs) != 0 || len(control.resumeReqs) != 0 || len(control.suspendReqs) != 0 { diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index 3cf41f8899..deea2f51ee 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -407,7 +407,7 @@ func (w *ActorWorkflow) ensureExternalSnapshotsReleased(ctx context.Context, act // wrote anything. func actorSnapshotStoragePrefix(actor *ateapi.Actor) (resources.StoragePrefix, error) { actorOwner := actorSnapshotOwner(actor) - if snap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); snap != nil { + if snap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED); snap != nil { snapshotURI := snap.GetDurableSnapshot().GetObject().GetSnapshotUri() uri, err := resources.ParseSnapshotURI(snapshotURI) if err != nil { @@ -423,7 +423,7 @@ func actorSnapshotStoragePrefix(actor *ateapi.Actor) (resources.StoragePrefix, e // Nothing of the actor's own is recorded. Unless a suspend died partway, // nothing was ever written under its prefix: the in-progress URI is // recorded before atelet uploads the first object. - inProgressSnap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + inProgressSnap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) inProgress := inProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() if inProgress == "" { return resources.StoragePrefix{}, nil diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index 5eb4abbba4..09543e72b5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -153,7 +153,7 @@ func (*detachDeletePlugin) DeleteVolume(context.Context, string) error { return // count as detached, and any other error fails the step as before. func TestRevertAndDelete_AssignedNodeGone(t *testing.T) { memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED for _, tc := range []struct { name string state ateapi.ActorState @@ -261,7 +261,7 @@ func TestRevertAndDelete_AssignedNodeGone(t *testing.T) { if s := got.GetStatus(); s.GetState() != ateapi.ActorState_ACTOR_STATE_SUSPENDED || s.GetAssignedNode() != "" { t.Errorf("after revert: state %v, assigned node %q, want SUSPENDED with no node", s.GetState(), s.GetAssignedNode()) } - if snaps := got.GetStatus().GetSnapshots(); len(snaps) != 1 || snaps[0].GetGeneration() != 1 || snaps[0].GetDurableSnapshot().GetStatus() != completed { + if snaps := got.GetStatus().GetSnapshots(); len(snaps) != 1 || snaps[0].GetGeneration() != 1 || snaps[0].GetDurableSnapshot().GetState() != completed { t.Errorf("after revert: snapshots %v, want only the completed durable snapshot at generation 1", snaps) } default: @@ -384,8 +384,8 @@ func TestEnsureExternalSnapshotsReleased(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 2 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, currentType, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, currentType, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) @@ -433,7 +433,7 @@ func TestEnsureExternalSnapshotsReleased_CollectsStrandedSnapshots(t *testing.T) actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", currentSnapshot.Name(), currentSnapshot.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", currentSnapshot.Name(), currentSnapshot.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } }) @@ -473,7 +473,7 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) { mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) @@ -506,7 +506,7 @@ func TestEnsureExternalSnapshotsReleased_DeletePrefixFailure(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } }) @@ -584,7 +584,7 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } }) } @@ -656,7 +656,7 @@ func TestDeleteActorWorkflow_RetryKeepsAdmission(t *testing.T) { observed := mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) // The guards a client sends after reading the actor. diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 8f43df3dfe..1f324ea860 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -641,7 +641,7 @@ func schedulingConstraints(actor *ateapi.Actor, tmpl *ateapi.ActorTemplate) sche // creation included), or nil when the actor has none and cold-boots. func snapshotToResume(status *ateapi.ActorStatus) *ateapi.Snapshot { return latestSnapshot(status, func(s *ateapi.Snapshot) bool { - return s.GetLocality() != "" || s.GetDurableSnapshot().GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return s.GetLocality() != "" || s.GetDurableSnapshot().GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED }) } @@ -683,7 +683,7 @@ func canRelaxRuntime(actor *ateapi.Actor, constraints scheduling.Constraints) bo if localElsewhere(snap, status.GetAssignedNode()) { return true } - return snap.GetDurableSnapshot().GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return snap.GetDurableSnapshot().GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } // appliesWithFallback reports whether worker satisfies constraints, or — when diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_detach.go b/cmd/ateapi/internal/controlapi/workflow_resume_detach.go index 3b9495995c..174d9a374d 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_detach.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_detach.go @@ -198,7 +198,7 @@ func localElsewhere(snap *ateapi.Snapshot, node string) bool { // no completed durable copy. func latestLocalNotDurable(actor *ateapi.Actor) bool { local := findLatestLocalSnapshot(actor.GetStatus()) - return local != nil && local.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return local != nil && local.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } func canDetachWithoutAtelet(ctx context.Context, actor *ateapi.Actor, err error) bool { @@ -217,7 +217,7 @@ func canDetachWithoutAtelet(ctx context.Context, actor *ateapi.Actor, err error) return false } local := findLatestLocalSnapshot(actor.GetStatus()) - return local != nil && local.GetDurableSnapshot().GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return local != nil && local.GetDurableSnapshot().GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } // detachActorError maps a DetachActor error to the one the resume fails with. diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go index e4848bcd57..9479df65e6 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_detach_test.go @@ -362,7 +362,7 @@ func TestDetachFromFullNode(t *testing.T) { snap := newPausedSnapshot(t, 1, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uuid, tt.node) snap.SandboxRuntime = tt.snapRuntime if tt.completed { - snap.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + snap.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } actor := seedActorWithSnapshot(t, ctx, st, suspended, tt.node, snap) w, atelet := newDetachTestWorkflow(t, ctx, st) @@ -390,7 +390,7 @@ func TestDetachFromFullNode(t *testing.T) { if err != nil { t.Errorf("GetActor: %v", err) } else if _, err := st.UpdateActor(ctx, actorRef, store.PreconditionFrom(cur), func(a *ateapi.Actor) error { - a.Status.Snapshots[0].DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + a.Status.Snapshots[0].DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED return nil }); err != nil { t.Errorf("UpdateActor: %v", err) @@ -481,7 +481,7 @@ func TestDetachFromFullNode(t *testing.T) { } got := after.GetStatus().GetSnapshots() if len(got) != 1 || got[0].GetLocality() != "" || - got[0].GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + got[0].GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { t.Errorf("snapshots = %v, want %s durable only and COMPLETED", got, uuid) } }) @@ -611,7 +611,7 @@ func TestResumeActor_NodeGoneWithPendingUpload(t *testing.T) { worker := seedRuntimeWorker(t, ctx, st, "pod-2", "node-2", workerRuntime) memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY durable := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", "durable-1", - someActorSnapshotURI(t, testStorageLocation, "team-a", "durable-1"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + someActorSnapshotURI(t, testStorageLocation, "team-a", "durable-1"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) pending := newPausedSnapshot(t, 2, memory, "", "pending-2", "node-3") if tt.otherRuntime { pending.SandboxRuntime = testRuntime("amd64", "") @@ -647,7 +647,7 @@ func TestResumeActor_NodeGoneWithPendingUpload(t *testing.T) { } // The restore proved the upload complete. snap := findSnapshotByGeneration(s.GetSnapshots(), 2) - if snap.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED || snap.GetLocality() != "" { + if snap.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED || snap.GetLocality() != "" { t.Errorf("generation 2 = %v, want COMPLETED with no locality", snap) } return @@ -749,7 +749,7 @@ func TestDetachFromFullNode_ReentryRecordsAfterConflict(t *testing.T) { t.Errorf("AssignedNode = %q, want none", got) } if got := stored.GetStatus().GetSnapshots(); len(got) != 1 || got[0].GetLocality() != "" || - got[0].GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + got[0].GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { t.Errorf("snapshots = %v, want pause-snap-1 durable only and COMPLETED", got) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index eb2e4a8072..0da9100150 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -255,18 +255,18 @@ func TestFinalizeRunning_UnrecordableUploadStillRuns(t *testing.T) { if got.GetStatus().GetState() != ateapi.ActorState_ACTOR_STATE_RUNNING { t.Errorf("state = %v, want RUNNING", got.GetStatus().GetState()) } - if snap := got.GetStatus().GetSnapshots()[0]; snap.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING || snap.GetLocality() != "node-3" { + if snap := got.GetStatus().GetSnapshots()[0]; snap.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING || snap.GetLocality() != "node-3" { t.Errorf("snapshot = %v, want it unrecorded: PENDING on node-3", snap) } } func TestRecordRestoredUpload(t *testing.T) { memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - snapshot := func(locality string, fidelity ateapi.SnapshotFidelity, storage ateapi.SnapshotStorageStatus) *ateapi.Snapshot { + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + snapshot := func(locality string, fidelity ateapi.SnapshotFidelity, storage ateapi.SnapshotStorageState) *ateapi.Snapshot { snap := newPausedSnapshot(t, 1, fidelity, "", "snap-1", locality) - snap.DurableSnapshot.Status = storage + snap.DurableSnapshot.State = storage return snap } for _, tt := range []struct { @@ -1436,7 +1436,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { unspecScope := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED extSnap := func(uri string, fidelity ateapi.SnapshotFidelity) *ateapi.Snapshot { - return newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", durableSnapshotName, uri, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + return newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", durableSnapshotName, uri, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) } localSnap := func(name string, fidelity ateapi.SnapshotFidelity) *ateapi.Snapshot { return newPausedSnapshot(t, 1, fidelity, "", name, "node-1") @@ -1451,7 +1451,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { uploadedLocalSnap := func(name string, fidelity ateapi.SnapshotFidelity) *ateapi.Snapshot { snap := localSnap(name, fidelity) snap.DurableSnapshot = &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: pauseUploadURI}, } return snap @@ -1525,7 +1525,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, } stampedExtSnap := func(owner ateapi.SnapshotOwner, uri string, fidelity ateapi.SnapshotFidelity, rt *ateapi.ActorSandboxRuntime) *ateapi.Snapshot { - snap := newDurableSnapshot(1, owner, fidelity, "", durableSnapshotName, uri, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap := newDurableSnapshot(1, owner, fidelity, "", durableSnapshotName, uri, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) snap.SandboxRuntime = rt return snap } @@ -2707,7 +2707,7 @@ func seedRuntimeWorker(t *testing.T, ctx context.Context, st store.Interface, po // generation 1 under team-a/id1's own prefix, stamped with runtime. func durableMemorySnapshot(t *testing.T, runtime *ateapi.ActorSandboxRuntime) *ateapi.Snapshot { t.Helper() - snap := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-1"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-1"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) snap.SandboxRuntime = runtime return snap } @@ -3005,7 +3005,7 @@ func TestValidateAssignedWorker_RuntimeFallback(t *testing.T) { // Re-entry must accept a previously allowed ROOTFS assignment. local := newPausedSnapshot(t, 1, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-arm") local.SandboxRuntime = testSandboxRuntime() - local.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + local.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED resumingActor := &ateapi.Actor{ Metadata: &ateapi.ResourceMetadata{Atespace: "team-a", Name: "id1", Uid: "own-actor-uid"}, Status: &ateapi.ActorStatus{ @@ -3047,11 +3047,11 @@ func TestRequiredSnapshotRuntime(t *testing.T) { memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY rootfs := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS volumes := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING - durable := func(gen int32, fidelity ateapi.SnapshotFidelity, capturedUnder string, status ateapi.SnapshotStorageStatus, rt *ateapi.ActorSandboxRuntime) *ateapi.Snapshot { - snap := newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, capturedUnder, "snap", "gs://bucket/snap", status) + durable := func(gen int32, fidelity ateapi.SnapshotFidelity, capturedUnder string, state ateapi.SnapshotStorageState, rt *ateapi.ActorSandboxRuntime) *ateapi.Snapshot { + snap := newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, capturedUnder, "snap", "gs://bucket/snap", state) snap.SandboxRuntime = rt return snap } @@ -3333,8 +3333,8 @@ func TestResume_ChecksMountedVolumesBeforeAttach(t *testing.T) { func TestCanRelaxRuntime(t *testing.T) { rt := testSandboxRuntime() memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING suspended := ateapi.ActorState_ACTOR_STATE_SUSPENDED resuming := ateapi.ActorState_ACTOR_STATE_RESUMING @@ -3344,9 +3344,9 @@ func TestCanRelaxRuntime(t *testing.T) { local := func(gen int32) *ateapi.Snapshot { return newPausedSnapshot(t, gen, memory, "", "local", "node-1") } - localDurable := func(gen int32, storageStatus ateapi.SnapshotStorageStatus) *ateapi.Snapshot { + localDurable := func(gen int32, storageState ateapi.SnapshotStorageState) *ateapi.Snapshot { snap := local(gen) - snap.GetDurableSnapshot().Status = storageStatus + snap.GetDurableSnapshot().State = storageState return snap } localNoDurable := func(gen int32) *ateapi.Snapshot { diff --git a/cmd/ateapi/internal/controlapi/workflow_revert.go b/cmd/ateapi/internal/controlapi/workflow_revert.go index 6c553fcd91..5c5ff604be 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert.go @@ -220,7 +220,7 @@ func (w *ActorWorkflow) ensureInProgressSnapshotDiscarded(ctx context.Context, a var inProgress []string for _, snap := range actor.GetStatus().GetSnapshots() { storage := snap.GetDurableSnapshot() - if storage.GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING && storage.GetObject().GetSnapshotUri() != "" { + if storage.GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING && storage.GetObject().GetSnapshotUri() != "" { inProgress = append(inProgress, storage.GetObject().GetSnapshotUri()) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_revert_test.go b/cmd/ateapi/internal/controlapi/workflow_revert_test.go index 40d247d296..e03e8eb48f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert_test.go @@ -146,7 +146,7 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { inProgress.DurableSnapshot.Object.SnapshotUri = mustActorSnapshotURI(t, tmpl, actor, "local-in-progress").String() s.LastAssignedGeneration = 3 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "kept", keptURI, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "kept", keptURI, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), local1, inProgress, } @@ -170,7 +170,7 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { if got := gotStatus.GetLastAssignedGeneration(); got != 3 { t.Errorf("last_assigned_generation = %d, want 3", got) } - gotSnap := findLatestDurableSnapshot(gotStatus, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(gotStatus, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if got := gotSnap.GetGeneration(); got != 1 { t.Errorf("retained snapshot generation = %d, want 1", got) } @@ -183,7 +183,7 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { if got := findLatestLocalSnapshot(gotStatus); got != nil { t.Errorf("local snapshot info = %v, want nil", got) } - if got := findLatestDurableSnapshot(gotStatus, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING); got != nil { + if got := findLatestDurableSnapshot(gotStatus, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING); got != nil { t.Errorf("in-progress durable snapshot = %v, want nil", got) } if got := len(gotStatus.GetSnapshots()); got != 1 { @@ -260,12 +260,12 @@ func TestRevertActor_UnreachableNode(t *testing.T) { s.AssignedNode = tt.node s.LastAssignedGeneration = 2 snap := newPausedSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "uploaded", tt.node) - snap.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + snap.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED if tt.pending { - snap.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + snap.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING } s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, snap.GetFidelity(), "", "older", "gs://snapshots/atespaces/team-a/actors/older/snapshots/snapshot", ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, snap.GetFidelity(), "", "older", "gs://snapshots/atespaces/team-a/actors/older/snapshots/snapshot", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), snap, } if tt.assigned { @@ -343,7 +343,7 @@ func TestRevertActor_UnreachableNode(t *testing.T) { // terminate, as in TestRevertActor_ReturnsActorToItsSnapshot. func TestRevertActor_PausedSnapshotAlreadyUploaded(t *testing.T) { memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED ctx := context.Background() st, cleanup := storetest.SetupTestStore(t) defer cleanup() @@ -365,7 +365,7 @@ func TestRevertActor_PausedSnapshotAlreadyUploaded(t *testing.T) { objects.PutSnapshot(t, uploaded, "manifest.json") mustUpdateActorStatus(t, ctx, st, actor, func(s *ateapi.ActorStatus) { paused := newPausedSnapshot(t, 2, memory, "", "pause-snap", "node-1") - paused.DurableSnapshot = &ateapi.SnapshotStorage{Status: completed, Object: &ateapi.ObjectSnapshot{SnapshotUri: uploaded.String()}} + paused.DurableSnapshot = &ateapi.SnapshotStorage{State: completed, Object: &ateapi.ObjectSnapshot{SnapshotUri: uploaded.String()}} s.LastAssignedGeneration = 2 s.Snapshots = []*ateapi.Snapshot{ newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", old.Name(), old.String(), completed), @@ -472,7 +472,7 @@ func TestRevertActor_NoSnapshotToRevertTo(t *testing.T) { if got := reverted.GetStatus().GetCrash(); got != nil { t.Errorf("Crash = %v, want cleared by the revert", got) } - gotSnap := findLatestDurableSnapshot(reverted.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(reverted.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != "" { t.Errorf("external snapshot = %q, want none", got) } @@ -581,15 +581,15 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, currentType, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, currentType, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.Name(), current.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } if tt.earlierPause { s.LastAssignedGeneration++ - s.Snapshots = append(s.Snapshots, newDurableSnapshot(s.LastAssignedGeneration, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", earlierPause.Name(), earlierPause.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)) + s.Snapshots = append(s.Snapshots, newDurableSnapshot(s.LastAssignedGeneration, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", earlierPause.Name(), earlierPause.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)) } if tt.inFlight != "" { s.LastAssignedGeneration++ - s.Snapshots = append(s.Snapshots, newDurableSnapshot(s.LastAssignedGeneration, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)) + s.Snapshots = append(s.Snapshots, newDurableSnapshot(s.LastAssignedGeneration, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.Name(), inFlight.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)) } }) @@ -647,7 +647,7 @@ func TestEnsureRevertedFinalized_NoObjectStore(t *testing.T) { AssignedNode: "node-1", LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "abandoned", someActorSnapshotURI(t, testStorageLocation, "team-a", "abandoned"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "abandoned", someActorSnapshotURI(t, testStorageLocation, "team-a", "abandoned"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, }) @@ -667,7 +667,7 @@ func TestEnsureRevertedFinalized_NoObjectStore(t *testing.T) { if got := finalized.GetStatus().GetAssignedNode(); got != "" { t.Errorf("assigned node = %q, want empty", got) } - if got := findLatestDurableSnapshot(finalized.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING); got != nil { + if got := findLatestDurableSnapshot(finalized.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING); got != nil { t.Errorf("in-progress durable snapshot = %v, want nil", got) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index 15a4720c37..2256218e49 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -194,7 +194,7 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res actorTemplate.GetMetadata().GetUid(), uri.Name(), uri.String(), - ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING, ) snap.SandboxRuntime = proto.CloneOf(toUpdate.GetStatus().GetWorkerAssignment().GetSandboxRuntime()) toUpdate.Status.Snapshots = append(toUpdate.Status.Snapshots, snap) @@ -401,9 +401,9 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r toUpdate.Status.Snapshots = applied.GetSnapshots() snap := snapshotAtLatestGeneration(toUpdate.Status) if mode == suspendSync { - snap.GetDurableSnapshot().Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + snap.GetDurableSnapshot().State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } - durable := snap.GetDurableSnapshot().GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + durable := snap.GetDurableSnapshot().GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED // Keep the actor on the node that holds its snapshot. atelet holds an // async suspend's snapshot until it detaches it, which it does only // once the snapshot is durable. @@ -450,14 +450,14 @@ func ateletKeptLocal(state *ateletpb.ActorSnapshotState, snap *ateapi.Snapshot) // whole prefix. An upload atelet reports can make a newer snapshot COMPLETED // outside any workflow, so the next one can drop more than one. func (w *ActorWorkflow) releaseDroppedSnapshots(ctx context.Context, before, after *ateapi.Actor) { - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED kept := map[string]bool{} for _, snap := range after.GetStatus().GetSnapshots() { kept[snap.GetDurableSnapshot().GetObject().GetSnapshotUri()] = true } for _, snap := range before.GetStatus().GetSnapshots() { dropped := snap.GetDurableSnapshot().GetObject().GetSnapshotUri() - if snap.GetDurableSnapshot().GetStatus() != completed || kept[dropped] { + if snap.GetDurableSnapshot().GetState() != completed || kept[dropped] { continue } if err := w.releaseReplacedSnapshot(ctx, before, dropped); err != nil { diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go index c37e83b30e..ff428eb461 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go @@ -235,7 +235,7 @@ func TestSuspendActorAsync_Finalized_RecordsFidelity(t *testing.T) { if localSnap.GetFidelity() != tc.want { t.Errorf("local Snapshot.Fidelity = %v, want %v", localSnap.GetFidelity(), tc.want) } - if got := localSnap.GetDurableSnapshot().GetStatus(); got != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { + if got := localSnap.GetDurableSnapshot().GetState(); got != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING { t.Errorf("local DurableSnapshot.Status = %v, want PENDING", got) } if got := localSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got == "" { @@ -358,7 +358,7 @@ func TestSuspendActorAsync_AteletPaused_DialFailureLeavesActorRetryable(t *testi var snapshots []*ateapi.Snapshot if tt.prevSnapshot != "" { - snapshots = append(snapshots, newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "prev-snapshot", tt.prevSnapshot, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + snapshots = append(snapshots, newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "prev-snapshot", tt.prevSnapshot, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)) } snapshots = append(snapshots, newInProgressSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "actor-1-never-written")) @@ -390,7 +390,7 @@ func TestSuspendActorAsync_AteletPaused_DialFailureLeavesActorRetryable(t *testi if gotSnap := snapshotAtLatestGeneration(stored.GetStatus()); gotSnap.GetUuid() != "actor-1-never-written" { t.Errorf("in-progress local snapshot uuid = %q, want preserved for debugging", gotSnap.GetUuid()) } - gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != tt.prevSnapshot { t.Errorf("SnapshotUri = %q, want %q", got, tt.prevSnapshot) } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 685b250736..335bd79162 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -108,7 +108,7 @@ func TestSuspendActor_CheckpointError(t *testing.T) { t.Errorf("worker allocated actors = %d, want 1", got) } pending := snapshotAtLatestGeneration(st) - if pending.GetUuid() == "" || pending.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { + if pending.GetUuid() == "" || pending.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING { t.Fatalf("snapshot = %v, want a persisted pending snapshot", pending) } first := atelet.checkpointRequest().GetSnapshot() @@ -169,7 +169,7 @@ func TestSuspendActor_MarkedSuspending_SnapshotURI(t *testing.T) { assertStoppedAtDial(t, err) marked := mustReadActor(t, ctx, persistence, actorRef) - inProgressSnap := findLatestDurableSnapshot(marked.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + inProgressSnap := findLatestDurableSnapshot(marked.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) uri := mustParseSnapshotURI(t, inProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri()) if !resources.IsValidResourceName(uri.Name()) { t.Errorf("in-progress snapshot name = %q, want a valid resource name", uri.Name()) @@ -193,15 +193,15 @@ func TestSuspendActor_MarkedSuspending_SnapshotURI(t *testing.T) { // the same shape, so the retry's mode may differ from the first attempt's. func TestSuspendActor_MarkedSuspending_ReentryKeepsPersistedSnapshotLocation(t *testing.T) { const firstAttempt = "gs://bucket/root/atespaces/team-a/actors/actor-uid/snapshots/first-attempt" - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED tests := []struct { name string mode suspendMode wantUploadMode ateletpb.UploadMode wantLocality string - wantDurable ateapi.SnapshotStorageStatus + wantDurable ateapi.SnapshotStorageState wantNode string }{ {"sync", suspendSync, ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE, "", completed, ""}, @@ -245,7 +245,7 @@ func TestSuspendActor_MarkedSuspending_ReentryKeepsPersistedSnapshotLocation(t * if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != firstAttempt { t.Errorf("committed durable snapshot URI = %q, want the first attempt's location", got) } - if got := gotSnap.GetDurableSnapshot().GetStatus(); got != tt.wantDurable { + if got := gotSnap.GetDurableSnapshot().GetState(); got != tt.wantDurable { t.Errorf("durable copy status = %v, want %v", got, tt.wantDurable) } if got := gotSnap.GetLocality(); got != tt.wantLocality { @@ -267,7 +267,7 @@ func TestSuspendActorWorkflow_SuspendedReturnsAsIs(t *testing.T) { withDurable := func(a *ateapi.Actor) { a.Status.LastAssignedGeneration = 1 a.Status.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "durable-snap", durableURI, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "durable-snap", durableURI, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } } @@ -337,7 +337,7 @@ func TestSuspendActor_MarkedSuspending_StateMatrix(t *testing.T) { } if seedState == ateapi.ActorState_ACTOR_STATE_RUNNING { latest := snapshotAtLatestGeneration(stored.GetStatus()) - if latest.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING || latest.GetLocality() != "" { + if latest.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING || latest.GetLocality() != "" { t.Errorf("state %v: latest snapshot = %v, want a PENDING durable copy and no locality", seedState, latest) } } @@ -430,9 +430,9 @@ func TestSuspendActor_AteletSuspended_DialFailureLeavesActorRetryable(t *testing seedSuspendTemplate(t, ctx, persistence, &ateapi.SnapshotConfig{StorageLocation: testStorageLocation}) var snapshots []*ateapi.Snapshot if tt.prevSnapshot != "" { - snapshots = append(snapshots, newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "prev-snapshot", tt.prevSnapshot, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + snapshots = append(snapshots, newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "prev-snapshot", tt.prevSnapshot, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)) } - snapshots = append(snapshots, newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "never-written", neverWritten, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING)) + snapshots = append(snapshots, newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "never-written", neverWritten, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING)) actor := &ateapi.Actor{ Metadata: &ateapi.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, @@ -465,10 +465,10 @@ func TestSuspendActor_AteletSuspended_DialFailureLeavesActorRetryable(t *testing if stored.GetStatus().GetState() != ateapi.ActorState_ACTOR_STATE_SUSPENDING { t.Errorf("state = %v, want unchanged SUSPENDING", stored.GetStatus().GetState()) } - if gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING); gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != neverWritten { + if gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING); gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != neverWritten { t.Errorf("in-progress durable snapshot URI = %q, want preserved for debugging", gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri()) } - gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != tt.prevSnapshot { t.Errorf("SnapshotUri = %q, want %q", got, tt.prevSnapshot) } @@ -499,7 +499,7 @@ func TestSuspendActor_AteletSuspended_WireRequest(t *testing.T) { WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-uid-1", snapshotName, snapshotURI, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-uid-1", snapshotName, snapshotURI, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, }) @@ -587,8 +587,8 @@ func TestSuspendActor_Finalized_ReleasesReplacedSnapshot(t *testing.T) { mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 2 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, prevType, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, prevType, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) @@ -598,7 +598,7 @@ func TestSuspendActor_Finalized_ReleasesReplacedSnapshot(t *testing.T) { } // Whichever way the previous snapshot went, the actor now owns the // one it just wrote and is no longer borrowing. - gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != fresh.String() { t.Errorf("external snapshot = %q, want the one this suspend wrote, %q", got, fresh) } @@ -620,8 +620,8 @@ func TestSuspendActor_Finalized_ReleasesReplacedSnapshot(t *testing.T) { func TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot(t *testing.T) { const node = "node-1" memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING for _, tc := range []struct { name string // reported records the upload before the suspend, as atelet's report @@ -731,8 +731,8 @@ func TestSnapshotLocationChange_LeavesNothingBehind(t *testing.T) { mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 2 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) @@ -796,8 +796,8 @@ func TestSuspendActor_Finalized_CommitsDespiteObjectStoreFailure(t *testing.T) { mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 2 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) @@ -809,11 +809,11 @@ func TestSuspendActor_Finalized_CommitsDespiteObjectStoreFailure(t *testing.T) { if got := stored.GetStatus().GetState(); got != ateapi.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got) } - gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotSnap := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if got := gotSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(); got != fresh.String() { t.Errorf("actor snapshot uri = %q, want the one this suspend wrote, %q", got, fresh) } - if got := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING); got != nil { + if got := findLatestDurableSnapshot(stored.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING); got != nil { t.Errorf("in-progress durable snapshot = %v, want cleared", got) } if len(objects.Snapshot(t, previous)) == 0 { @@ -854,8 +854,8 @@ func TestSuspendActor_Finalized_KeepsReplacedSnapshotOnConflict(t *testing.T) { mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 2 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), - newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.Name(), previous.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), + newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.Name(), fresh.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } }) @@ -901,7 +901,7 @@ func TestSuspendActor_Finalized_KeepsCrashFromWorkerDelete(t *testing.T) { WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, }) @@ -952,7 +952,7 @@ func TestSuspendActor_Finalized_ReleasesOnlyOwnWorker(t *testing.T) { WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, } @@ -1000,7 +1000,7 @@ func TestSuspendActor_Finalized_MissingDurableSnapshotClearsAssignment(t *testin WorkerAssignment: wireTestAssignment(), LastAssignedGeneration: 1, Snapshots: []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), }, }, }) @@ -1091,7 +1091,7 @@ type snapshotOutcome struct { fidelity ateapi.SnapshotFidelity locality string // durable is the durable copy's status, UNSPECIFIED when there is none. - durable ateapi.SnapshotStorageStatus + durable ateapi.SnapshotStorageState } // TestSuspendAndPause_CharacteristicMatrix pins, for each actor shape either @@ -1101,15 +1101,15 @@ type snapshotOutcome struct { func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { const node = "node-1" volumes := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES - durable := func(gen int32, status ateapi.SnapshotStorageStatus) *ateapi.Snapshot { + durable := func(gen int32, state ateapi.SnapshotStorageState) *ateapi.Snapshot { name := fmt.Sprintf("snap-%d", gen) - return newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, volumes, "", name, someActorSnapshotURI(t, testStorageLocation, "team-a", name), status) + return newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, volumes, "", name, someActorSnapshotURI(t, testStorageLocation, "team-a", name), state) } local := func(gen int32, locality string) *ateapi.Snapshot { return newPausedSnapshot(t, gen, volumes, "", fmt.Sprintf("snap-%d", gen), locality) } - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING ateletVolumes := ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES immediate := ateletpb.UploadMode_UPLOAD_MODE_IMMEDIATE @@ -1363,7 +1363,7 @@ func describeSnapshots(status *ateapi.ActorStatus) []snapshotOutcome { generation: snap.GetGeneration(), fidelity: snap.GetFidelity(), locality: snap.GetLocality(), - durable: snap.GetDurableSnapshot().GetStatus(), + durable: snap.GetDurableSnapshot().GetState(), }) } return out @@ -1401,11 +1401,11 @@ func assertRequestNamesLatestSnapshot(t *testing.T, atelet *capturingAtelet, tmp func TestSuspendActor_AppliesCheckpointState(t *testing.T) { const node = "node-1" memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - pending := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - durable := func(gen int32, status ateapi.SnapshotStorageStatus) *ateapi.Snapshot { + pending := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + durable := func(gen int32, state ateapi.SnapshotStorageState) *ateapi.Snapshot { name := fmt.Sprintf("snap-%d", gen) - return newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", name, someActorSnapshotURI(t, testStorageLocation, "team-a", name), status) + return newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, memory, "", name, someActorSnapshotURI(t, testStorageLocation, "team-a", name), state) } report := func(gen int32, uri string, fidelity ateletpb.SnapshotFidelity) *ateletpb.SnapshotReport { return &ateletpb.SnapshotReport{Snapshot: &ateletpb.Snapshot{ @@ -1594,7 +1594,7 @@ func TestSuspendActor_AppliesCheckpointState(t *testing.T) { func TestSuspendActor_Finalized_RecordsLocalCopyWhenKept(t *testing.T) { const node = "node-1" memory := ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - completed := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + completed := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED localReport := func(snap *ateletpb.Snapshot) *ateletpb.ActorSnapshotState { return &ateletpb.ActorSnapshotState{LatestLocal: &ateletpb.SnapshotReport{Snapshot: snap}} } diff --git a/cmd/ateapi/internal/controlapi/workflow_tag.go b/cmd/ateapi/internal/controlapi/workflow_tag.go index 25e50e2c8e..ba20448362 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag.go @@ -194,7 +194,7 @@ func (w *ActorWorkflow) ensureTagSnapshotReleased(ctx context.Context, tag *atea return nil } tagRef := resources.TagRefFromTag(tag) - snapshotURI := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED) + snapshotURI := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED) if snapshotURI == "" { markSkipped(ctx, "the tag records no snapshot") return nil @@ -293,7 +293,7 @@ func (w *ActorWorkflow) ensureTagSourceSnapshotDurable(ctx context.Context, acto defer func() { err = done(err) }() snap := latestSnapshot(actor.GetStatus(), func(*ateapi.Snapshot) bool { return true }) - if snap.GetDurableSnapshot().GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + if snap.GetDurableSnapshot().GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { markSkipped(ctx, "the actor's latest snapshot is already durable") return actor, nil } @@ -387,7 +387,7 @@ func promoteSnapshotError(ctx context.Context, err error) error { // sourceSnapshotForTag returns the actor's latest COMPLETED external snapshot, // the one a tag copies. func sourceSnapshotForTag(actorRef resources.ActorRef, actor *ateapi.Actor) (*ateapi.Snapshot, error) { - snap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap := findLatestDurableSnapshot(actor.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if snap == nil { return nil, apierror.FailedPrecondition("Actor %s holds no external snapshot to tag", actorRef) } @@ -465,7 +465,7 @@ func (w *ActorWorkflow) ensureTagSnapshotRecorded(ctx context.Context, tag *atea "", dst.Name(), dst.String(), - ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING, ) pending.SandboxRuntime = proto.CloneOf(snapshot.GetSandboxRuntime()) stored, err := w.store.UpdateTag(ctx, tagRef, store.PreconditionFrom(tag), func(toUpdate *ateapi.Tag) error { @@ -523,7 +523,7 @@ func (w *ActorWorkflow) ensureTagFinalized(ctx context.Context, tag *ateapi.Tag, "", dst.Name(), dst.String(), - ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, ) finalSnapshot.SandboxRuntime = proto.CloneOf(snapshot.GetSandboxRuntime()) stored, err := w.store.UpdateTag(ctx, tagRef, store.PreconditionFrom(tag), func(toUpdate *ateapi.Tag) error { diff --git a/cmd/ateapi/internal/controlapi/workflow_tag_test.go b/cmd/ateapi/internal/controlapi/workflow_tag_test.go index b1e14ec14c..e82a75213e 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag_test.go @@ -50,7 +50,7 @@ func seedTagSource(t *testing.T, ctx context.Context, persistence store.Interfac objects.PutSnapshot(t, uri, objectNames...) actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { s.LastAssignedGeneration = 1 - snap := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, template.GetMetadata().GetUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap := newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, template.GetMetadata().GetUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) snap.SandboxRuntime = testSandboxRuntime() s.Snapshots = []*ateapi.Snapshot{snap} }) @@ -94,7 +94,7 @@ func TestTagActorSnapshot(t *testing.T) { t.Fatalf("TagActorSnapshot: %v", err) } - tagSnapshot := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + tagSnapshot := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if tagSnapshot == "" || tagSnapshot == actorSnapshot.String() { t.Fatalf("tag snapshot uri = %q, want a copy of its own rather than the actor's %q", tagSnapshot, actorSnapshot) } @@ -214,7 +214,7 @@ func TestTagActorSnapshot_Preconditions(t *testing.T) { } s.LastAssignedGeneration = 1 s.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, uri.Name(), uri.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, uri.Name(), uri.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), } }) } @@ -274,7 +274,7 @@ func TestTagActorSnapshot_TagsNewestExternalSnapshot(t *testing.T) { uri := mustActorSnapshotURI(t, template, actor, "actor-1-snapshot") objects.PutSnapshot(t, uri, "manifest.json") mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapi.ActorStatus) { - snap := newDurableSnapshot(tt.extGen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, template.GetMetadata().GetUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap := newDurableSnapshot(tt.extGen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, template.GetMetadata().GetUid(), uri.Name(), uri.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) snap.Locality = tt.locality s.LastAssignedGeneration = tt.lastAssigned s.Snapshots = []*ateapi.Snapshot{snap} @@ -458,9 +458,9 @@ func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { }) pausedURI := mustActorSnapshotURI(t, template, actor, "paused") objects.PutSnapshot(t, pausedURI, "manifest.json", "memory.zst") - pausedStatus := ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + pausedStatus := ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING if tt.uploaded { - pausedStatus = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + pausedStatus = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } paused := newDurableSnapshot(2, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, pausedURI.Name(), pausedURI.String(), pausedStatus) if !tt.nodeLost { @@ -474,7 +474,7 @@ func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { if tt.olderDurable { olderURI := mustActorSnapshotURI(t, template, actor, "older") objects.PutSnapshot(t, olderURI, "manifest.json") - s.Snapshots = append(s.Snapshots, newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, olderURI.Name(), olderURI.String(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + s.Snapshots = append(s.Snapshots, newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, olderURI.Name(), olderURI.String(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED)) } s.Snapshots = append(s.Snapshots, proto.CloneOf(paused)) }) @@ -554,7 +554,7 @@ func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { return } - tagSnapshot := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + tagSnapshot := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if tt.nodeLost { // Nothing to upload from: the actor is left as it was, and the // tag copied the older durable snapshot. @@ -569,7 +569,7 @@ func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { // The local snapshot is now durable and stays on its node. wantPaused := proto.CloneOf(paused) - wantPaused.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + wantPaused.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED if diff := cmp.Diff(wantPaused, findSnapshotByGeneration(stored.GetStatus().GetSnapshots(), 2), protocmp.Transform()); diff != "" { t.Errorf("the actor's latest snapshot mismatch (-want +got):\n%s", diff) } @@ -612,7 +612,7 @@ func TestTagActorSnapshot_RecreateAfterCopyFailure(t *testing.T) { if err != nil { t.Fatalf("GetTag after the external store failure: %v", err) } - if got := tagDurableSnapshotURI(pending, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != "" { + if got := tagDurableSnapshotURI(pending, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED); got != "" { t.Errorf("snapshot uri after the failure = %q, want unset: the copy never finished", got) } strandedURI := mustReservedTagSnapshotURI(t, pending) @@ -651,7 +651,7 @@ func TestTagActorSnapshot_RecreateAfterCopyFailure(t *testing.T) { if err != nil { t.Fatalf("TagActorSnapshot after the delete: %v", err) } - recreated := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + recreated := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if recreated == "" || recreated == stranded { t.Fatalf("snapshot uri after the delete = %q, want a fresh prefix rather than the stranded %q", recreated, stranded) } @@ -732,7 +732,7 @@ func TestTagActorSnapshot_RecreateAfterRecordFailure(t *testing.T) { if err != nil { t.Fatalf("TagActorSnapshot after the delete: %v", err) } - if got, want := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), mustTagSnapshotURI(t, template, "team-a", tag.GetMetadata().GetUid()).String(); got != want { + if got, want := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), mustTagSnapshotURI(t, template, "team-a", tag.GetMetadata().GetUid()).String(); got != want { t.Errorf("recreated tag snapshot uri = %q, want UID-based URI %q", got, want) } } @@ -782,7 +782,7 @@ func TestTagActorSnapshot_RacesDelete(t *testing.T) { // Nothing was collected out from under the copy, and the row in the store it // is still there. - if got := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != pendingURI { + if got := tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED); got != pendingURI { t.Errorf("snapshot uri = %q, want the prefix the create reserved, %q", got, pendingURI) } if diff := cmp.Diff([]string{"manifest.json", "memory.zst"}, objects.Snapshot(t, mustParseSnapshotURI(t, pendingURI))); diff != "" { @@ -967,7 +967,7 @@ func TestDeleteTag_NotFound(t *testing.T) { func mustReservedTagSnapshotURI(t *testing.T, tag *ateapi.Tag) resources.SnapshotURI { t.Helper() - uri, err := resources.ParseSnapshotURI(tagDurableSnapshotURI(tag, ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED)) + uri, err := resources.ParseSnapshotURI(tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_UNSPECIFIED)) if err != nil { t.Fatalf("ParseSnapshotURI: %v", err) } diff --git a/cmd/ateapi/internal/controlapi/workflow_test.go b/cmd/ateapi/internal/controlapi/workflow_test.go index d8e50ce33b..0c43928fd8 100644 --- a/cmd/ateapi/internal/controlapi/workflow_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_test.go @@ -299,7 +299,7 @@ func TestActorStateChangeRecordSkippedOnConflict(t *testing.T) { if _, err := persistence.UpdateActor(ctx, actorRef, store.PreconditionFrom(stale), func(toUpdate *ateapi.Actor) error { toUpdate.Status.LastAssignedGeneration = 1 toUpdate.Status.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "someone-else", someActorSnapshotURI(t, testStorageLocation, "team-a", "someone-else"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "someone-else", someActorSnapshotURI(t, testStorageLocation, "team-a", "someone-else"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING), } return nil }); err != nil { diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index ae01c8c43f..d98d2d3653 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -180,7 +180,7 @@ func withUploadedPausedShape(t *testing.T, node string) func(*ateapi.Actor) { paused := withPausedShape(t, node) return func(a *ateapi.Actor) { paused(a) - a.Status.Snapshots[0].DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + a.Status.Snapshots[0].DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED } } @@ -189,7 +189,7 @@ func withUploadedPausedShape(t *testing.T, node string) func(*ateapi.Actor) { // yet. func newInProgressSnapshot(t *testing.T, gen int32, fidelity ateapi.SnapshotFidelity, uuid string) *ateapi.Snapshot { t.Helper() - return newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", uuid, someActorSnapshotURI(t, testStorageLocation, "team-a", uuid), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + return newDurableSnapshot(gen, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", uuid, someActorSnapshotURI(t, testStorageLocation, "team-a", uuid), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) } // newPausedSnapshot builds the snapshot an asynchronous suspend leaves on node diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go index aeed776291..afbf559580 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go @@ -115,7 +115,7 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { actor := seedAPIActor(t, ctx, persistence, ateapi.ActorState_ACTOR_STATE_RUNNING, func(a *ateapi.Actor) { a.Status.LastAssignedGeneration = 2 a.Status.Snapshots = []*ateapi.Snapshot{ - newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "last", someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "last", someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED), newInProgressSnapshot(t, 2, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "partial-snapshot"), } }) @@ -140,12 +140,12 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { } // The in-progress checkpoint is kept: it names the prefix whatever atelet // already uploaded lives under, which delete or revert needs to collect it. - gotInProgressSnap := findLatestDurableSnapshot(got.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) + gotInProgressSnap := findLatestDurableSnapshot(got.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING) if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot"); gotInProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != want { t.Errorf("in-progress external checkpoint not preserved: %v", got.GetStatus()) } // The last completed snapshot is what makes the actor resumable, so it stays. - gotCompletedSnap := findLatestDurableSnapshot(got.GetStatus(), ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + gotCompletedSnap := findLatestDurableSnapshot(got.GetStatus(), ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"); gotCompletedSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() != want { t.Errorf("external snapshot = %q, want it preserved as %q", gotCompletedSnap.GetDurableSnapshot().GetObject().GetSnapshotUri(), want) } diff --git a/cmd/ateapi/internal/store/atepg/tag.go b/cmd/ateapi/internal/store/atepg/tag.go index d5cca85516..2028fb93a9 100644 --- a/cmd/ateapi/internal/store/atepg/tag.go +++ b/cmd/ateapi/internal/store/atepg/tag.go @@ -195,12 +195,12 @@ func isAllowedTagSnapshotTransition(stored, mutated *ateapi.Snapshot) bool { if mutated == nil || stored.GetDurableSnapshot() == nil || mutated.GetDurableSnapshot() == nil { return false } - if stored.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING || - mutated.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + if stored.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING || + mutated.GetDurableSnapshot().GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { return false } expected := proto.CloneOf(stored) - expected.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + expected.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED return proto.Equal(expected, mutated) } diff --git a/cmd/ateapi/internal/store/atepg/tag_test.go b/cmd/ateapi/internal/store/atepg/tag_test.go index 1b65f7ef0f..0ab2c69815 100644 --- a/cmd/ateapi/internal/store/atepg/tag_test.go +++ b/cmd/ateapi/internal/store/atepg/tag_test.go @@ -40,7 +40,7 @@ func createTestTag(t *testing.T, s *Persistence, tagAtespace, tagName string) *a Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://bucket/atespaces/" + tagAtespace + "/tags/" + tagName}, }, }, diff --git a/cmd/ateapi/internal/store/storecontract/contract.go b/cmd/ateapi/internal/store/storecontract/contract.go index 7ff8970cf5..81838ce7e5 100644 --- a/cmd/ateapi/internal/store/storecontract/contract.go +++ b/cmd/ateapi/internal/store/storecontract/contract.go @@ -1050,7 +1050,7 @@ func runActorContractTests(t *testing.T, setup func(t *testing.T) store.Interfac Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: testActorSnapshotURI("gs://bucket", testAtespace, "snapshot-1")}, }, }}, @@ -1067,7 +1067,7 @@ func runActorContractTests(t *testing.T, setup func(t *testing.T) store.Interfac Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: testActorSnapshotURI("gs://bucket", testAtespace, "snapshot-2")}, }, }}, @@ -1360,7 +1360,7 @@ func newTestSuspendedActor(atespace, name string) *ateapi.Actor { Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: testActorSnapshotURI("gs://private", atespace, name)}, }, }}, @@ -1383,7 +1383,7 @@ func newTestInProgressTag(name string, actor *ateapi.Actor) *ateapi.Tag { Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING, Object: &ateapi.ObjectSnapshot{ SnapshotUri: testTagSnapshotURI("gs://private", atespace, name), }, @@ -1431,8 +1431,8 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) } // A reserved tag has a PENDING snapshot storage entry and is not // usable until the copy lands and finalize marks it COMPLETED. - if got := tag.GetStatus().GetSnapshot().GetDurableSnapshot().GetStatus(); got != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING { - t.Errorf("reserved tag snapshot storage status = %v, want PENDING", got) + if got := tag.GetStatus().GetSnapshot().GetDurableSnapshot().GetState(); got != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING { + t.Errorf("reserved tag snapshot storage state = %v, want PENDING", got) } if diff := cmp.Diff(inProgressTag, tag, protocmp.Transform(), ignoreUID, ignoreVersion, ignoreTimestamps); diff != "" { t.Errorf("stored tag mismatch (-want +got):\n%s", diff) @@ -1453,8 +1453,8 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) if gotURI != wantURI { t.Errorf("finalized tag snapshot uri = %q, want %q", gotURI, wantURI) } - if got := ready.GetStatus().GetSnapshot().GetDurableSnapshot().GetStatus(); got != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { - t.Errorf("finalized tag snapshot storage status = %v, want COMPLETED", got) + if got := ready.GetStatus().GetSnapshot().GetDurableSnapshot().GetState(); got != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { + t.Errorf("finalized tag snapshot storage state = %v, want COMPLETED", got) } updated, err := s.UpdateTag(ctx, resources.TagRef{Atespace: "team-a", Name: "production"}, store.PreconditionFrom(ready), func(toUpdate *ateapi.Tag) error { @@ -1789,7 +1789,7 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) // finalizeTag mutates a reserved tag the way the tag workflow's second // transaction does: it marks the pending snapshot storage COMPLETED. func finalizeTag(toUpdate *ateapi.Tag) error { - toUpdate.Status.Snapshot.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + toUpdate.Status.Snapshot.DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED return nil } diff --git a/cmd/ateapi/internal/workerservice/jwt_test.go b/cmd/ateapi/internal/workerservice/jwt_test.go index 355594142a..f84c6939fd 100644 --- a/cmd/ateapi/internal/workerservice/jwt_test.go +++ b/cmd/ateapi/internal/workerservice/jwt_test.go @@ -132,7 +132,7 @@ func pausedStatus(state ateapi.ActorState, node string) *ateapi.ActorStatus { func uploadPendingStatus(node string) *ateapi.ActorStatus { status := pausedStatus(ateapi.ActorState_ACTOR_STATE_SUSPENDED, node) status.Snapshots[0].DurableSnapshot = &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://snaps/root/atespaces/team-a/actors/uid/snap-1"}, } return status @@ -361,7 +361,7 @@ func TestMintWorkerActorJwt_Errors(t *testing.T) { ctx: authed, status: func() *ateapi.ActorStatus { status := uploadPendingStatus("node-1") - status.Snapshots[0].DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + status.Snapshots[0].DurableSnapshot.State = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED return status }(), want: codes.OK, diff --git a/cmd/kubectl-ate/internal/cmd/tag_test.go b/cmd/kubectl-ate/internal/cmd/tag_test.go index a6ae387316..104b1dca81 100644 --- a/cmd/kubectl-ate/internal/cmd/tag_test.go +++ b/cmd/kubectl-ate/internal/cmd/tag_test.go @@ -63,7 +63,7 @@ func TestUpdateTagScope(t *testing.T) { Snapshot: &ateapi.Snapshot{ Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://private/atespaces/space-1/tags/tag-1"}, }, }, diff --git a/cmd/kubectl-ate/internal/printer/printer.go b/cmd/kubectl-ate/internal/printer/printer.go index 8d418c46c9..cc94eff9c9 100644 --- a/cmd/kubectl-ate/internal/printer/printer.go +++ b/cmd/kubectl-ate/internal/printer/printer.go @@ -359,7 +359,7 @@ func PrintTagsTo(out io.Writer, tags []*ateapi.Tag, format string) error { func durableSnapshotStorage(snapshot *ateapi.Snapshot) *ateapi.SnapshotStorage { st := snapshot.GetDurableSnapshot() - if st.GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED && + if st.GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED && st.GetObject().GetSnapshotUri() != "" { return st } diff --git a/cmd/kubectl-ate/internal/printer/printer_test.go b/cmd/kubectl-ate/internal/printer/printer_test.go index bcbf0a3bfd..19bb9c1a0e 100644 --- a/cmd/kubectl-ate/internal/printer/printer_test.go +++ b/cmd/kubectl-ate/internal/printer/printer_test.go @@ -596,7 +596,7 @@ func TestPrintTagsTo_Table(t *testing.T) { Snapshot: &ateapi.Snapshot{ Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v2"}, }, }, @@ -613,7 +613,7 @@ func TestPrintTagsTo_Table(t *testing.T) { Snapshot: &ateapi.Snapshot{ Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v1"}, }, }, @@ -632,7 +632,7 @@ func TestPrintTagsTo_Table(t *testing.T) { Snapshot: &ateapi.Snapshot{ Fidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING, Object: &ateapi.ObjectSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v3"}, }, }, diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 25bb180b90..1da963a945 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -744,7 +744,7 @@ func durableSnapshot(status *ateapi.ActorStatus) *ateapi.Snapshot { var best *ateapi.Snapshot for _, snap := range status.GetSnapshots() { st := snap.GetDurableSnapshot() - if st.GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED && + if st.GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED && st.GetObject().GetSnapshotUri() != "" { if best == nil || snap.GetGeneration() > best.GetGeneration() { best = snap diff --git a/internal/e2e/suites/security/crossatespace_test.go b/internal/e2e/suites/security/crossatespace_test.go index c0590122f3..1597fadcaf 100644 --- a/internal/e2e/suites/security/crossatespace_test.go +++ b/internal/e2e/suites/security/crossatespace_test.go @@ -467,7 +467,7 @@ func durableSnapshotURI(status *ateapi.ActorStatus) string { func snapshotURI(snap *ateapi.Snapshot) string { st := snap.GetDurableSnapshot() - if st.GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + if st.GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { return st.GetObject().GetSnapshotUri() } return "" diff --git a/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go b/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go index 502690aa07..0b9af6acf8 100644 --- a/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go +++ b/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go @@ -199,7 +199,7 @@ func expectGCS(t *testing.T, ctx context.Context, what, token, method, uri, body func durableURI(t *testing.T, actor *ateapi.Actor, owner ateapi.SnapshotOwner) string { t.Helper() for _, snap := range actor.GetStatus().GetSnapshots() { - if snap.GetOwner() == owner && snap.GetDurableSnapshot().GetStatus() == ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + if snap.GetOwner() == owner && snap.GetDurableSnapshot().GetState() == ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { return snap.GetDurableSnapshot().GetObject().GetSnapshotUri() } } diff --git a/internal/objectstoreplugin/actorscope/actorscope.go b/internal/objectstoreplugin/actorscope/actorscope.go index 98ffc9b966..79d64addc5 100644 --- a/internal/objectstoreplugin/actorscope/actorscope.go +++ b/internal/objectstoreplugin/actorscope/actorscope.go @@ -103,7 +103,7 @@ func FromActor(actor *ateapi.Actor) ([]Grant, error) { case ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR: // Its own prefix is already granted above. case ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG: - if durable.GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + if durable.GetState() != ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED { continue } add(parsed.OwnerPrefix(), ReadOnly) diff --git a/internal/objectstoreplugin/actorscope/actorscope_test.go b/internal/objectstoreplugin/actorscope/actorscope_test.go index 2cec8b1191..e031113f1e 100644 --- a/internal/objectstoreplugin/actorscope/actorscope_test.go +++ b/internal/objectstoreplugin/actorscope/actorscope_test.go @@ -34,10 +34,10 @@ func actorURI(location, name string) string { return location + "/atespaces/" + atespace + "/actors/" + uid + "/snapshots/" + name } -func gen(n int32, owner ateapi.SnapshotOwner, status ateapi.SnapshotStorageStatus, uri string) *ateapi.Snapshot { +func gen(n int32, owner ateapi.SnapshotOwner, state ateapi.SnapshotStorageState, uri string) *ateapi.Snapshot { s := &ateapi.Snapshot{Generation: n, Owner: owner} if uri != "" { - s.DurableSnapshot = &ateapi.SnapshotStorage{Status: status, Object: &ateapi.ObjectSnapshot{SnapshotUri: uri}} + s.DurableSnapshot = &ateapi.SnapshotStorage{State: state, Object: &ateapi.ObjectSnapshot{SnapshotUri: uri}} } else { s.Locality = "node-1" } @@ -65,8 +65,8 @@ var ( tagURI = loc + "/atespaces/ate-golden/tags/" + tagUID tagGrant = Grant{Prefix: prefix(tagURI), Access: ReadOnly} actorGrant = Grant{Prefix: prefix(loc + "/atespaces/" + atespace + "/actors/" + uid), Access: ReadWrite} - completed = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED - pending = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING + completed = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED + pending = ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING ownerActor = ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR ownerTag = ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG ) diff --git a/internal/objectstoreplugin/exchange/exchange_test.go b/internal/objectstoreplugin/exchange/exchange_test.go index e3ced082e8..42976167b7 100644 --- a/internal/objectstoreplugin/exchange/exchange_test.go +++ b/internal/objectstoreplugin/exchange/exchange_test.go @@ -179,7 +179,7 @@ func actorWithTag() *ateapi.Actor { Status: &ateapi.ActorStatus{State: ateapi.ActorState_ACTOR_STATE_RUNNING, AssignedNode: "node-1", Snapshots: []*ateapi.Snapshot{{ Generation: 1, Owner: ateapi.SnapshotOwner_SNAPSHOT_OWNER_TAG, DurableSnapshot: &ateapi.SnapshotStorage{ - Status: ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + State: ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapi.ObjectSnapshot{SnapshotUri: tagURI}, }, }}}, From c57124f6608430885394c80a3fb1777266b2a460 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 27 Oct 2026 16:51:47 +0000 Subject: [PATCH 11/53] actornet: pin the sysctls a sandbox namespace inherits from the node (#3310) Why is this needed? (Or link the issue. Open one first for features, API, install or CI changes.) FYI @destijl, follow-up on https://github.com/agent-substrate/substrate/issues/2939#issuecomment-6296019529 Besides forwarding, a new namespace picks up the node's: - `accept_redirects` - `route_localnet` - `rp_filter` - `accept_local` - `accept_source_route` - ... and has IPv6 on with `accept_ra=1`. NOTE: `SetSysctls` now writes every key, since turning forwarding off sets `conf/all/accept_redirects` back to 1. The sysctls are set before any link is added, a conf/default write doesn't reach existing links for keys the node set explicitly. Fixes #2939 AI use: Assisted with identifying the gaps (with upstream sources cited), drafting the change and testing. I read through this before submitting and iterated a bit. - [x] Tests pass - [x] Docs updated Signed-off-by: Benjamin Elder --- internal/ateworker/actornet/sandbox.go | 93 ++++++++++++------ .../ateworker/actornet/sandbox_linux_test.go | 98 +++++++++++++++++++ internal/netns/sysctl.go | 56 +++-------- internal/netns/sysctl_linux_test.go | 49 ++++++---- 4 files changed, 200 insertions(+), 96 deletions(-) diff --git a/internal/ateworker/actornet/sandbox.go b/internal/ateworker/actornet/sandbox.go index 58403e57f8..67e020a31a 100644 --- a/internal/ateworker/actornet/sandbox.go +++ b/internal/ateworker/actornet/sandbox.go @@ -98,6 +98,9 @@ func SetupSandboxNetwork(ctx context.Context, cfg SandboxNetworkConfig) (_ *Sand _ = netns.RemoveNamed(actorNSName) } }() + if err := netns.Do(ctx, actorNS, prepareNamespace); err != nil { + return nil, err + } // Without a veth, atunnel shares the namespace with the runtime's tap. atunnelNS := actorNS @@ -114,7 +117,7 @@ func SetupSandboxNetwork(ctx context.Context, cfg SandboxNetworkConfig) (_ *Sand }() atunnelNS = outer } - if err := setupGatewaySide(ctx, atunnelNS, cfg.EgressPort); err != nil { + if err := installEgressRules(atunnelNS, cfg.EgressPort); err != nil { return nil, err } @@ -125,6 +128,59 @@ func SetupSandboxNetwork(ctx context.Context, cfg SandboxNetworkConfig) (_ *Sand }, nil } +// prepareNamespace pins sandboxSysctls and brings up lo. It must run before +// the namespace has any link but lo: a link copies conf/default when it is +// created, and a later write to conf/default does not reach links whose +// settings the node had set explicitly. +func prepareNamespace(context.Context) error { + if err := netns.SetSysctls(sandboxSysctls...); err != nil { + return err + } + // lo carries the namespace's own traffic. gVisor builds the actor's + // loopback from it. + return linkUp("lo") +} + +// sandboxSysctls are set in every sandbox namespace, in this order. A new +// namespace copies its IPv4 conf/all and conf/default from the node, where +// kube-proxy and CNI plugins set their own values, and takes the kernel's +// IPv6 defaults. Neither suits a link with an untrusted actor on the other end. +var sandboxSysctls = []netns.Sysctl{ + // atunnel answers the actor's DNS on 53, and the worker holds no + // CAP_NET_BIND_SERVICE. Per-namespace, so it grants nothing outside. + {Key: "net/ipv4/ip_unprivileged_port_start", Value: "0"}, + // Nothing in a sandbox namespace routes. The forward chain in + // installEgressRules backs this up. + {Key: "net/ipv4/conf/all/forwarding", Value: "0"}, + {Key: "net/ipv4/conf/default/forwarding", Value: "0"}, + {Key: "net/ipv6/conf/all/forwarding", Value: "0"}, + {Key: "net/ipv6/conf/default/forwarding", Value: "0"}, + // conf/all/accept_redirects defaults to 1 and the kernel sets it back to + // 1 whenever conf/all/forwarding changes to 0, so it comes after + // forwarding. With forwarding off the kernel takes ICMP redirects when + // either conf/all or the interface allows them. + {Key: "net/ipv4/conf/all/accept_redirects", Value: "0"}, + {Key: "net/ipv4/conf/default/accept_redirects", Value: "0"}, + // kube-proxy's iptables mode enables this on nodes. It would deliver + // actor packets addressed to 127.0.0.0/8 to sockets in the gateway + // namespace. + {Key: "net/ipv4/conf/all/route_localnet", Value: "0"}, + {Key: "net/ipv4/conf/default/route_localnet", Value: "0"}, + // Drop packets from the actor that claim a gateway address as their + // source, carry a source route, or come from outside the /30. + {Key: "net/ipv4/conf/all/accept_local", Value: "0"}, + {Key: "net/ipv4/conf/default/accept_local", Value: "0"}, + {Key: "net/ipv4/conf/all/accept_source_route", Value: "0"}, + {Key: "net/ipv4/conf/default/accept_source_route", Value: "0"}, + {Key: "net/ipv4/conf/all/rp_filter", Value: "1"}, + {Key: "net/ipv4/conf/default/rp_filter", Value: "1"}, + // Don't take router advertisements or redirects from the actor. IPv6 + // reads both only from the link's own copy, which it takes from + // conf/default when the link is created. + {Key: "net/ipv6/conf/default/accept_ra", Value: "0"}, + {Key: "net/ipv6/conf/default/accept_redirects", Value: "0"}, +} + // setupVethPair creates the gateway namespace and the veth pair joining it to // actorNS. The caller owns the returned handle and its name. func setupVethPair(ctx context.Context, cfg SandboxNetworkConfig, actorNS netns.Handle) (_ netns.Handle, retErr error) { @@ -141,7 +197,10 @@ func setupVethPair(ctx context.Context, cfg SandboxNetworkConfig, actorNS netns. }() // Keep the kernel-owned peer outside gVisor's namespace. - if err := netns.Do(ctx, outer, func(context.Context) error { + if err := netns.Do(ctx, outer, func(ctx context.Context) error { + if err := prepareNamespace(ctx); err != nil { + return err + } veth := &netlink.Veth{ LinkAttrs: netlink.LinkAttrs{Name: gatewayVethName}, PeerName: ActorVethName, @@ -171,14 +230,6 @@ func setupVethPair(ctx context.Context, cfg SandboxNetworkConfig, actorNS netns. // gVisor imports these addresses and routes into its network stack. if err := netns.Do(ctx, actorNS, func(context.Context) error { - // The runtime namespace has no reason to route. - if err := netns.DisableForwarding(); err != nil { - return err - } - // Loopback lets the actor reach its own address. - if err := linkUp("lo"); err != nil { - return err - } eth0, err := netlink.LinkByName(ActorVethName) if err != nil { return err @@ -207,28 +258,6 @@ func linkUp(name string) error { return netlink.LinkSetUp(link) } -// setupGatewaySide brings up lo and puts atunnel in front of the actor's TCP. -func setupGatewaySide(ctx context.Context, ns netns.Handle, egressPort uint16) error { - if err := netns.Do(ctx, ns, func(context.Context) error { - // atunnel answers the actor's DNS on 53 (requiring unprivileged port bind), - // and actor packets must never route. Configure both under a single - // /proc/sys remount. - if err := netns.SetSysctls( - netns.Sysctl{Key: "net/ipv4/ip_unprivileged_port_start", Value: "0"}, - netns.Sysctl{Key: "net/ipv4/conf/all/forwarding", Value: "0"}, - netns.Sysctl{Key: "net/ipv4/conf/default/forwarding", Value: "0"}, - netns.Sysctl{Key: "net/ipv6/conf/all/forwarding", Value: "0"}, - netns.Sysctl{Key: "net/ipv6/conf/default/forwarding", Value: "0"}, - ); err != nil { - return err - } - return linkUp("lo") - }); err != nil { - return err - } - return installEgressRules(ns, egressPort) -} - // installEgressRules rejects link-local traffic before it reaches the egress PEP, // then redirects other TCP egress to atunnel. The actor's own /30 is excluded // from both rules so ingress replies and DNS reach their local sockets. diff --git a/internal/ateworker/actornet/sandbox_linux_test.go b/internal/ateworker/actornet/sandbox_linux_test.go index eaebedd3fd..8f3381a7a4 100644 --- a/internal/ateworker/actornet/sandbox_linux_test.go +++ b/internal/ateworker/actornet/sandbox_linux_test.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "io/fs" "net" "net/http" "net/url" @@ -486,6 +487,103 @@ func TestSandboxNamespacesDoNotForward(t *testing.T) { } } +// A new namespace copies conf/all and conf/default from the node, and a link +// copies conf/default when it is created. Both shapes end up with the pinned +// values on every link they add. +func TestSandboxNamespacesDoNotInheritNodeSettings(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + + // Settings the kernel reads from the link's own copy, alone or combined + // with conf/all by OR or max, so a link seeded from the node's + // conf/default would undo them. rp_filter=2 on a link wins over all=1. + perLink := map[string]string{ + "forwarding": "0", + "accept_redirects": "0", + "route_localnet": "0", + "accept_local": "0", + "rp_filter": "1", + } + // IPv6 reads these only from the link's own copy. + perLinkV6 := map[string]string{ + "forwarding": "0", + "accept_ra": "0", + "accept_redirects": "0", + } + for _, tc := range []struct { + name string + uid string + veth bool + }{ + {"veth", "a0a0a0a0-0000-0000-0000-00000000000a", true}, + {"tap", "b0b0b0b0-0000-0000-0000-00000000000b", false}, + } { + t.Run(tc.name, func(t *testing.T) { + n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ActorUID: tc.uid, Veth: tc.veth, EgressPort: testEgressPort}) + if err != nil { + t.Fatalf("SetupSandboxNetwork: %v", err) + } + t.Cleanup(func() { CleanupSandboxNetwork(n) }) + + // A link added after setup, as the microVM tap is. + if err := netns.Do(ctx, n.GatewayNetNS, func(context.Context) error { + if err := netlink.LinkAdd(&netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: "late0"}, PeerName: "late1"}); err != nil { + return err + } + for _, name := range []string{"late0", "late1"} { + if err := linkUp(name); err != nil { + return err + } + } + return nil + }); err != nil { + t.Fatalf("adding a late link: %v", err) + } + + for _, ns := range []netns.Handle{n.RuntimeNetNS, n.GatewayNetNS} { + if err := netns.Do(ctx, ns, func(context.Context) error { + want := map[string]string{} + for _, s := range sandboxSysctls { + want[s.Key] = s.Value + } + links, err := netlink.LinkList() + if err != nil { + return err + } + for _, l := range links { + name := l.Attrs().Name + // lo predates the sysctls. + if l.Attrs().Flags&net.FlagLoopback != 0 { + continue + } + for key, value := range perLink { + want["net/ipv4/conf/"+name+"/"+key] = value + } + for key, value := range perLinkV6 { + want["net/ipv6/conf/"+name+"/"+key] = value + } + } + for key, value := range want { + b, err := os.ReadFile("/proc/sys/" + key) + if errors.Is(err, fs.ErrNotExist) && strings.HasPrefix(key, "net/ipv6/") { + continue + } + if err != nil { + return err + } + if got := strings.TrimSpace(string(b)); got != value { + t.Errorf("%s = %s, want %s", key, got, value) + } + } + return nil + }); err != nil { + t.Fatalf("reading sysctls: %v", err) + } + } + }) + } +} + // Adds a second link to the gateway namespace and checks that actor UDP is not // routed onto it, even after the test turns forwarding back on. Deleting the // forward chain at the end confirms the probe would see a routed packet. diff --git a/internal/netns/sysctl.go b/internal/netns/sysctl.go index 1649ab037a..200434fe4e 100644 --- a/internal/netns/sysctl.go +++ b/internal/netns/sysctl.go @@ -33,31 +33,16 @@ type Sysctl struct { Value string } -// forwardingSysctls disables IPv4 and IPv6 forwarding across all interfaces -// and default interface templates. Explicitly clearing default/forwarding -// ensures interfaces created after namespace setup (such as microVM taps) -// do not inherit forwarding on nodes where all=0 but default=1. -var forwardingSysctls = []Sysctl{ - {Key: "net/ipv4/conf/all/forwarding", Value: "0"}, - {Key: "net/ipv4/conf/default/forwarding", Value: "0"}, - {Key: "net/ipv6/conf/all/forwarding", Value: "0"}, - {Key: "net/ipv6/conf/default/forwarding", Value: "0"}, -} - -// DisableForwarding turns off IPv4 and IPv6 forwarding in this namespace, which -// otherwise inherits the node's IPv4 setting. Missing IPv6 keys are skipped. -func DisableForwarding() error { - return SetSysctls(forwardingSysctls...) -} - -// SetSysctls writes value to each sysctl in the current network namespace under -// a single lock hold and at most one /proc/sys read-write remount. Missing -// net/ipv6/ keys are skipped. +// SetSysctls writes each sysctl, in order, in the current network namespace, +// under a single lock hold and at most one /proc/sys read-write remount. +// Every key is written even if it already reads that way, because writing one +// key can change another: turning net/ipv4/conf/all/forwarding off sets +// net/ipv4/conf/all/accept_redirects. Missing net/ipv6/ keys are skipped. func SetSysctls(sysctls ...Sysctl) error { return hostProcSys.set(sysctls) } -// procSysMu serializes setSysctl. Sandboxes are set up concurrently and share +// procSysMu serializes SetSysctls. Sandboxes are set up concurrently and share // one /proc/sys mount, so otherwise one caller's read-only restore can land // between another's remount and write, failing that write with EROFS. var procSysMu sync.Mutex @@ -65,13 +50,11 @@ var procSysMu sync.Mutex // procSys wraps the /proc/sys file and mount operations so that unit tests can // substitute a fake mount. Production code uses hostProcSys. type procSys struct { - readFile func(path string) ([]byte, error) writeFile func(path string, data []byte) error remount func(readOnly bool) error } var hostProcSys = procSys{ - readFile: os.ReadFile, writeFile: func(path string, data []byte) error { return os.WriteFile(path, data, 0o644) }, @@ -88,35 +71,20 @@ func (p procSys) set(sysctls []Sysctl) error { procSysMu.Lock() defer procSysMu.Unlock() - var toWrite []Sysctl - for _, s := range sysctls { - path := "/proc/sys/" + s.Key - b, err := p.readFile(path) - if err == nil && strings.TrimSpace(string(b)) == s.Value { - continue - } - if errors.Is(err, fs.ErrNotExist) { - if strings.HasPrefix(s.Key, "net/ipv6/") { + writeAll := func() error { + for _, s := range sysctls { + err := p.writeFile("/proc/sys/"+s.Key, []byte(s.Value+"\n")) + if errors.Is(err, fs.ErrNotExist) && strings.HasPrefix(s.Key, "net/ipv6/") { continue } - return fmt.Errorf("while setting %s in the current netns: %w", s.Key, err) - } - toWrite = append(toWrite, s) - } - if len(toWrite) == 0 { - return nil - } - - writeAll := func() error { - for _, s := range toWrite { - path := "/proc/sys/" + s.Key - if err := p.writeFile(path, []byte(s.Value+"\n")); err != nil { + if err != nil { return fmt.Errorf("while setting %s in the current netns: %w", s.Key, err) } } return nil } + // Only EROFS is worth remounting for; any other error is returned as is. err := writeAll() if err == nil || !errors.Is(err, unix.EROFS) { return err diff --git a/internal/netns/sysctl_linux_test.go b/internal/netns/sysctl_linux_test.go index c8a6da0f19..b9f3b0bdc2 100644 --- a/internal/netns/sysctl_linux_test.go +++ b/internal/netns/sysctl_linux_test.go @@ -75,6 +75,15 @@ func TestSetSysctlSerializesTheRemount(t *testing.T) { } } +// forwardingSysctls span both families, so a kernel without IPv6 exercises +// the missing-key path. +var forwardingSysctls = []Sysctl{ + {Key: "net/ipv4/conf/all/forwarding", Value: "0"}, + {Key: "net/ipv4/conf/default/forwarding", Value: "0"}, + {Key: "net/ipv6/conf/all/forwarding", Value: "0"}, + {Key: "net/ipv6/conf/default/forwarding", Value: "0"}, +} + // Setting multiple sysctls performs at most one read-write remount cycle. func TestSetSysctlsBatchesRemount(t *testing.T) { fake := &fakeProcSys{readOnly: true, values: map[string]string{ @@ -106,7 +115,7 @@ func TestSetSysctlsBatchesRemount(t *testing.T) { } } -func TestDisableForwardingClearsEveryFamily(t *testing.T) { +func TestSetSysctlsWritesEveryFamily(t *testing.T) { fake := &fakeProcSys{readOnly: true, values: map[string]string{ "/proc/sys/net/ipv4/conf/all/forwarding": "1", "/proc/sys/net/ipv4/conf/default/forwarding": "1", @@ -123,18 +132,12 @@ func TestDisableForwardingClearsEveryFamily(t *testing.T) { } } -func TestDisableIPv4ForwardingWhenIPv6Disabled(t *testing.T) { +func TestSetSysctlsSkipsMissingIPv6Keys(t *testing.T) { values := map[string]string{} p := procSys{ - readFile: func(path string) ([]byte, error) { - if strings.HasPrefix(path, "/proc/sys/net/ipv6/") { - return nil, &os.PathError{Op: "open", Path: path, Err: unix.ENOENT} - } - return []byte("1\n"), nil - }, writeFile: func(path string, data []byte) error { if strings.HasPrefix(path, "/proc/sys/net/ipv6/") { - t.Errorf("wrote missing key %s", path) + return &os.PathError{Op: "open", Path: path, Err: unix.ENOENT} } values[path] = strings.TrimSpace(string(data)) return nil @@ -154,10 +157,9 @@ func TestDisableIPv4ForwardingWhenIPv6Disabled(t *testing.T) { } } -func TestDisableForwardingReportsAMissingIPv4Key(t *testing.T) { +func TestSetSysctlsReportsAMissingIPv4Key(t *testing.T) { p := procSys{ - readFile: func(path string) ([]byte, error) { return nil, &os.PathError{Op: "open", Path: path, Err: unix.ENOENT} }, - writeFile: func(path string, data []byte) error { return nil }, + writeFile: func(path string, data []byte) error { return &os.PathError{Op: "open", Path: path, Err: unix.ENOENT} }, remount: func(bool) error { return nil }, } if err := p.set(forwardingSysctls); !errors.Is(err, unix.ENOENT) { @@ -165,12 +167,26 @@ func TestDisableForwardingReportsAMissingIPv4Key(t *testing.T) { } } +// A key that already reads as wanted is still written: writing +// conf/all/forwarding changes conf/all/accept_redirects, so a value read +// before the batch says nothing about the value after it. +func TestSetSysctlsWritesKeysThatAlreadyMatch(t *testing.T) { + fake := &fakeProcSys{values: map[string]string{"/proc/sys/net/ipv4/conf/all/accept_redirects": "0"}} + if err := fake.procSys().set([]Sysctl{{Key: "net/ipv4/conf/all/accept_redirects", Value: "0"}}); err != nil { + t.Fatalf("set: %v", err) + } + if fake.writes != 1 { + t.Errorf("writes = %d, want 1", fake.writes) + } +} + // fakeProcSys is a read-only /proc/sys mount shared by all callers. Its // read-write remount is slow, so unserialized callers overlap inside it. type fakeProcSys struct { mu sync.Mutex readOnly bool values map[string]string + writes int // remounted counts callers inside the read-write window; maxRemounted is // its peak. remounted, maxRemounted, remountCalls int @@ -178,14 +194,6 @@ type fakeProcSys struct { func (f *fakeProcSys) procSys() procSys { return procSys{ - readFile: func(path string) ([]byte, error) { - f.mu.Lock() - defer f.mu.Unlock() - if v, ok := f.values[path]; ok { - return []byte(v + "\n"), nil - } - return []byte("1024\n"), nil - }, writeFile: func(path string, data []byte) error { f.mu.Lock() defer f.mu.Unlock() @@ -193,6 +201,7 @@ func (f *fakeProcSys) procSys() procSys { return &os.PathError{Op: "open", Path: path, Err: unix.EROFS} } f.values[path] = strings.TrimSpace(string(data)) + f.writes++ return nil }, remount: func(readOnly bool) error { From 4b40b2aa6b8738c3f1fa563fbda627c981c13f71 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Tue, 27 Oct 2026 17:10:05 +0000 Subject: [PATCH 12/53] benchmarking/nighthawk-egress: create namespaces in both deploy scripts (#3315) Following the README order on a fresh cluster, `deploy.sh` fails: Error from server (NotFound): error when creating ".../nighthawk-server.yaml": namespaces "nighthawk" not found https://github.com/agent-substrate/substrate/pull/2656 moved `nighthawk-server` and `traffic-driver-invoker` into namespace `nighthawk`, but only `deploy-traffic-driver.sh` (README step 2) applied the Namespace, through the WorkerPool template. The Namespace moves to `namespace.yaml` and both scripts apply it, so either script works on its own. Neither `--delete` path removes it, since the namespace is shared between the two scripts; the README says to delete it by hand after `deploy-traffic-driver.sh --delete`. The README target URLs, the port-forward namespace, the `deploy.sh` messages, and the manifest comments that still said `ate-system` now say `nighthawk`. AI use: AI-assisted. An agent found the failure while running the benchmark and drafted the change and this description. I wrote the commit message and reviewed both. - [ ] Tests pass (`bash -n` and the boilerplate check pass locally; the end-to-end rerun on a fresh kind cluster is pending) - [x] Docs updated Signed-off-by: Yuan Gao --- benchmarking/nighthawk-egress/README.md | 14 ++++++++------ .../nighthawk-egress/deploy-traffic-driver.sh | 4 ++++ benchmarking/nighthawk-egress/deploy.sh | 11 +++++++---- benchmarking/nighthawk-egress/namespace.yaml | 18 ++++++++++++++++++ .../nighthawk-egress/nighthawk-server.yaml | 6 +++--- .../nighthawk-egress/traffic-driver.yaml.tmpl | 7 ------- 6 files changed, 40 insertions(+), 20 deletions(-) create mode 100644 benchmarking/nighthawk-egress/namespace.yaml diff --git a/benchmarking/nighthawk-egress/README.md b/benchmarking/nighthawk-egress/README.md index df62df9284..f01faaf57a 100644 --- a/benchmarking/nighthawk-egress/README.md +++ b/benchmarking/nighthawk-egress/README.md @@ -79,7 +79,7 @@ variable when deploying ATE. In a separate terminal, forward local port `19980` to the `traffic-driver-invoker` service: ```bash -kubectl -n ate-system port-forward svc/traffic-driver-invoker 19980:19980 +kubectl -n nighthawk port-forward svc/traffic-driver-invoker 19980:19980 ``` ## Running the Benchmark @@ -87,7 +87,7 @@ kubectl -n ate-system port-forward svc/traffic-driver-invoker 19980:19980 Run `nighthawk_client` against the port-forwarded `traffic-driver-invoker` endpoint (`grpc://127.0.0.1:19980`), which routes the gRPC request to `nighthawk_service` inside the `nighthawk/tester` actor: ```bash -docker run --rm --network=host --entrypoint /usr/local/bin/nighthawk_client envoyproxy/nighthawk-dev:latest --nighthawk-service grpc://127.0.0.1:19980 --rps 1000 --concurrency 1 --protocol http1 --duration 60 http://nighthawk-server.ate-system.svc:80 +docker run --rm --network=host --entrypoint /usr/local/bin/nighthawk_client envoyproxy/nighthawk-dev:latest --nighthawk-service grpc://127.0.0.1:19980 --rps 1000 --concurrency 1 --protocol http1 --duration 60 http://nighthawk-server.nighthawk.svc:80 ``` See [Nighthawk CLI help](https://github.com/envoyproxy/nighthawk#using-the-nighthawk-client-cli) for all command line options. @@ -96,9 +96,9 @@ See [Nighthawk CLI help](https://github.com/envoyproxy/nighthawk#using-the-night | Egress Mode | Target URL | Port | |---|---|---| -| Plaintext HTTP | `http://nighthawk-server.ate-system.svc:80` | `80` | -| HTTPS (TLS MITM) | `https://nighthawk-server.ate-system.svc:443` | `443` | -| TLS Passthrough | `https://nighthawk-server.ate-system.svc:8443` | `8443` | +| Plaintext HTTP | `http://nighthawk-server.nighthawk.svc:80` | `80` | +| HTTPS (TLS MITM) | `https://nighthawk-server.nighthawk.svc:443` | `443` | +| TLS Passthrough | `https://nighthawk-server.nighthawk.svc:8443` | `8443` | ## Cleanup @@ -108,8 +108,10 @@ Delete the `nighthawk/tester` actor, actor template, atespace, and worker pool: ./benchmarking/nighthawk-egress/deploy-traffic-driver.sh --delete ``` -Delete `nighthawk-server` and `traffic-driver-invoker` from `ate-system`: +Delete `nighthawk-server` and `traffic-driver-invoker` from `nighthawk`: ```bash ./benchmarking/nighthawk-egress/deploy.sh --delete ``` + +Neither script deletes the `nighthawk` namespace. Delete it with `kubectl delete namespace nighthawk` after `deploy-traffic-driver.sh --delete`, because the actor, template, and atespace live in the ate API, not in the namespace. diff --git a/benchmarking/nighthawk-egress/deploy-traffic-driver.sh b/benchmarking/nighthawk-egress/deploy-traffic-driver.sh index af5698742f..825cf3dd2c 100755 --- a/benchmarking/nighthawk-egress/deploy-traffic-driver.sh +++ b/benchmarking/nighthawk-egress/deploy-traffic-driver.sh @@ -29,6 +29,7 @@ ATESPACE="nighthawk" TEMPLATE_NAME="traffic-driver" ACTOR_NAME="tester" POOL_MANIFEST="${SCRIPT_DIR}/traffic-driver.yaml.tmpl" +NAMESPACE_MANIFEST="${SCRIPT_DIR}/namespace.yaml" MANIFEST="${SCRIPT_DIR}/traffic-driver-template.yaml.tmpl" EGRESS_POLICY_MANIFEST="${SCRIPT_DIR}/egress-policy.yaml" KUBECTL_ATE_BIN="kubectl-ate" @@ -159,6 +160,9 @@ deploy() { fi export KO_DOCKER_REPO + echo "Deploying namespace from ${NAMESPACE_MANIFEST}..." + run_kubectl apply -f "${NAMESPACE_MANIFEST}" + echo "Deploying worker pool from ${POOL_MANIFEST}..." hack/run-tool.sh ko resolve -f "${POOL_MANIFEST}" | run_kubectl apply -f - echo "Waiting for worker pool ${ATESPACE}/${TEMPLATE_NAME} to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." diff --git a/benchmarking/nighthawk-egress/deploy.sh b/benchmarking/nighthawk-egress/deploy.sh index f10f2d2441..d0a0f086b4 100755 --- a/benchmarking/nighthawk-egress/deploy.sh +++ b/benchmarking/nighthawk-egress/deploy.sh @@ -17,6 +17,7 @@ set -o errexit -o nounset -o pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +NAMESPACE_MANIFEST="${SCRIPT_DIR}/namespace.yaml" SERVER_MANIFEST="${SCRIPT_DIR}/nighthawk-server.yaml" INVOKER_MANIFEST="${SCRIPT_DIR}/traffic-driver-invoker.yaml" WAIT_TIMEOUT_SECS=120 @@ -26,8 +27,8 @@ usage() { echo "Usage: $0 [options]" echo "" echo "Options:" - echo " --deploy Deploy nighthawk-server and traffic-driver-invoker in ate-system (default)" - echo " --delete Delete nighthawk-server and traffic-driver-invoker from ate-system" + echo " --deploy Deploy nighthawk-server and traffic-driver-invoker in nighthawk (default)" + echo " --delete Delete nighthawk-server and traffic-driver-invoker from nighthawk" echo " --context CONTEXT kubectl context to use (default: \$KUBECTL_CONTEXT or current context)" echo " --wait-timeout SECONDS Rollout wait timeout in seconds (default: ${WAIT_TIMEOUT_SECS})" echo " -h, --help Show this help message" @@ -80,13 +81,15 @@ if [[ -n "${CONTEXT}" ]]; then fi if [[ "${action}" == "deploy" ]]; then + echo "Deploying namespace from ${NAMESPACE_MANIFEST}..." + "${kubectl_cmd[@]}" apply -f "${NAMESPACE_MANIFEST}" echo "Deploying Nighthawk server from ${SERVER_MANIFEST}..." "${kubectl_cmd[@]}" apply -f "${SERVER_MANIFEST}" echo "Deploying traffic-driver-invoker from ${INVOKER_MANIFEST}..." "${kubectl_cmd[@]}" apply -f "${INVOKER_MANIFEST}" - echo "Waiting for deployment/nighthawk-server in ate-system to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." + echo "Waiting for deployment/nighthawk-server in nighthawk to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." "${kubectl_cmd[@]}" -n nighthawk rollout status deployment/nighthawk-server --timeout="${WAIT_TIMEOUT_SECS}s" - echo "Waiting for deployment/traffic-driver-invoker in ate-system to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." + echo "Waiting for deployment/traffic-driver-invoker in nighthawk to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." "${kubectl_cmd[@]}" -n nighthawk rollout status deployment/traffic-driver-invoker --timeout="${WAIT_TIMEOUT_SECS}s" elif [[ "${action}" == "delete" ]]; then echo "Deleting Nighthawk server and traffic-driver-invoker..." diff --git a/benchmarking/nighthawk-egress/namespace.yaml b/benchmarking/nighthawk-egress/namespace.yaml new file mode 100644 index 0000000000..22f70c340f --- /dev/null +++ b/benchmarking/nighthawk-egress/namespace.yaml @@ -0,0 +1,18 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: nighthawk diff --git a/benchmarking/nighthawk-egress/nighthawk-server.yaml b/benchmarking/nighthawk-egress/nighthawk-server.yaml index 1f1cc7f3ec..ee5a2ed061 100644 --- a/benchmarking/nighthawk-egress/nighthawk-server.yaml +++ b/benchmarking/nighthawk-egress/nighthawk-server.yaml @@ -15,7 +15,7 @@ # In-cluster Envoy Nighthawk test server used as the origin target for egress # benchmarks. Exposes plaintext HTTP on :80 and :8080 and TLS-terminated HTTPS # on :443 and :8443 using a serving certificate issued by the cluster's -# servicedns pod-certificate signer (SAN: nighthawk-server.ate-system.svc). +# servicedns pod-certificate signer (SAN: nighthawk-server.nighthawk.svc). apiVersion: v1 kind: ConfigMap metadata: @@ -297,8 +297,8 @@ spec: podcert.ate.dev/canarying: live path: trust-bundle.pem --- -# Selecting Service in ate-system required by servicedns.podcert.ate.dev/identity -# to mint a serving certificate with SAN nighthawk-server.ate-system.svc. +# Selecting Service in nighthawk required by servicedns.podcert.ate.dev/identity +# to mint a serving certificate with SAN nighthawk-server.nighthawk.svc. apiVersion: v1 kind: Service metadata: diff --git a/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl b/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl index 6853b65fbb..7d622b79f1 100644 --- a/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl +++ b/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl @@ -17,13 +17,6 @@ # traffic-driver-template.yaml.tmpl and is created through the ate API with # `kubectl-ate create actor-template`. -apiVersion: v1 -kind: Namespace -metadata: - name: nighthawk - ---- - apiVersion: ate.dev/v1 kind: WorkerPool metadata: From 1f31ae0ff40361953fa6df465a205b6f2c1bed49 Mon Sep 17 00:00:00 2001 From: Chenyi Wang Date: Tue, 27 Oct 2026 17:27:11 +0000 Subject: [PATCH 13/53] An actor's new egress connections no longer fail with 403 an hour after it starts (#3144) `acquireConn` kept pooling HTTP/2 connections to `atenet-egress` after `MintWorkerCertificate` renewed the actor certificate. Because the gateway authenticates each `CONNECT` with the certificate the TLS handshake presented, every new tunnel on an old connection failed with 403 once the first certificate expired (#3142). `runDial` now records the `*tls.Certificate` the handshake presented on `h2ClientConn`. When `GetClientCertificate` returns a different certificate, `acquireConn` drops the connection from `h2Conns` and `retire` installs a `SetStateHook` that closes it once `InFlight` reaches 0. The switch happens at renewal (90% of the 1-hour lifetime), so existing connections have the remaining 10% to drain. Fixes #3142 AI use: AI-assisted to draft the fix and unit test. I reproduced on GKE, reviewed, and tested it. - [x] Tests pass - [ ] Docs updated --------- Signed-off-by: Chenyi Wang --- internal/atunnel/egress/client.go | 76 +++++++++++++++++++-- internal/atunnel/egress/client_test.go | 91 ++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 7 deletions(-) diff --git a/internal/atunnel/egress/client.go b/internal/atunnel/egress/client.go index d33db9167a..4c8242d952 100644 --- a/internal/atunnel/egress/client.go +++ b/internal/atunnel/egress/client.go @@ -16,6 +16,7 @@ package egress import ( "bufio" + "bytes" "context" "crypto/tls" "crypto/x509" @@ -201,7 +202,7 @@ func (c *Client) acquireConn(ctx context.Context) (*h2ClientConn, bool, *tls.Con } // First check if there is an existing H/2 connection with available stream capacity. - // TODO(yanavlasov): do not use existing H/2 connections if their keys have been rotated. + current, _ := c.tlsConfig.GetClientCertificate(nil) c.mu.Lock() if c.closed { c.mu.Unlock() @@ -217,6 +218,17 @@ func (c *Client) acquireConn(ctx context.Context) (*h2ClientConn, bool, *tls.Con if h2Conn.isClosed() { continue } + // The gateway authenticates each CONNECT with the handshake + // certificate, so retire connections from before a renewal. + if !h2Conn.retired.Load() && !sameCertificate(h2Conn.cert, current) { + h2Conn.retire() + } + if h2Conn.retired.Load() { + if !h2Conn.isClosed() { + keep = append(keep, h2Conn) + } + continue + } if reserved == nil { if err := h2Conn.cc.Reserve(); err == nil { reserved = h2Conn @@ -280,8 +292,23 @@ func (c *Client) acquireConn(ctx context.Context) (*h2ClientConn, bool, *tls.Con return nil, false, nil, err } if h2Conn != nil { + current, _ := c.tlsConfig.GetClientCertificate(nil) + c.mu.Lock() + if !h2Conn.retired.Load() && !sameCertificate(h2Conn.cert, current) { + h2Conn.retire() + } wasVerified := h2Conn.verified.Load() - if err := h2Conn.cc.Reserve(); err == nil { + retired := h2Conn.retired.Load() + var reserveErr error + if !retired { + reserveErr = h2Conn.cc.Reserve() + } + c.mu.Unlock() + if retired { + // A renewal retired it while it was being dialed. + continue + } + if reserveErr == nil { return h2Conn, wasVerified, nil, nil } if !wasVerified && h2Conn.isClosed() { @@ -327,12 +354,19 @@ func (c *Client) runDial(dialCtx context.Context, call *dialCall) { if dialErr != nil { err = fmt.Errorf("egress: connecting to egress gateway: %w", dialErr) } else { - tlsConn := tls.Client(rawConn, c.tlsConfig.Clone()) + tlsConfig := c.tlsConfig.Clone() + var presented *tls.Certificate + tlsConfig.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { + cert, err := c.tlsConfig.GetClientCertificate(info) + presented = cert + return cert, err + } + tlsConn := tls.Client(rawConn, tlsConfig) if hsErr := tlsConn.HandshakeContext(dialCtx); hsErr != nil { _ = rawConn.Close() err = fmt.Errorf("%w: %w", ErrGatewayHandshake, hsErr) } else if tlsConn.ConnectionState().NegotiatedProtocol == "h2" { - h2Conn, err = c.newH2ClientConn(dialCtx, tlsConn) + h2Conn, err = c.newH2ClientConn(dialCtx, tlsConn, presented) } else { h1Conn = tlsConn } @@ -371,7 +405,7 @@ func (c *Client) runDial(dialCtx context.Context, call *dialCall) { c.mu.Unlock() } -func (c *Client) newH2ClientConn(ctx context.Context, tlsConn *tls.Conn) (*h2ClientConn, error) { +func (c *Client) newH2ClientConn(ctx context.Context, tlsConn *tls.Conn, cert *tls.Certificate) (*h2ClientConn, error) { raw := &trackedTLSConn{Conn: tlsConn} var protocols http.Protocols protocols.SetHTTP2(true) @@ -394,11 +428,22 @@ func (c *Client) newH2ClientConn(ctx context.Context, tlsConn *tls.Conn) (*h2Cli return &h2ClientConn{ cc: cc, raw: raw, + cert: cert, localAddr: tlsConn.LocalAddr(), remoteAddr: tlsConn.RemoteAddr(), }, nil } +func sameCertificate(a, b *tls.Certificate) bool { + if a == b { + return true + } + if a == nil || b == nil || len(a.Certificate) == 0 || len(b.Certificate) == 0 { + return false + } + return bytes.Equal(a.Certificate[0], b.Certificate[0]) +} + func (c *Client) removeH2Conn(target *h2ClientConn) { c.mu.Lock() defer c.mu.Unlock() @@ -480,7 +525,7 @@ func (c *Client) connectH2(ctx context.Context, h2Conn *h2ClientConn, wasVerifie _ = h2Conn.cc.Close() } } - if wasVerified && ctx.Err() == nil && (h2Conn.isClosed() || h2Conn.cc.Available() == 0) { + if (wasVerified || h2Conn.retired.Load()) && ctx.Err() == nil && (h2Conn.isClosed() || h2Conn.cc.Available() == 0) { c.removeH2Conn(h2Conn) return nil, true, nil } @@ -518,19 +563,36 @@ type h2ClientConn struct { // H/2 connection. cc *http.ClientConn // TLS connection backing this H/2 connection. - raw *trackedTLSConn + raw *trackedTLSConn + // Client certificate presented during the TLS handshake. + cert *tls.Certificate localAddr net.Addr remoteAddr net.Addr // Set to true when at least one request had been completed. // Used to distinguish between connection establishment errors and errors // on an already working connection. verified atomic.Bool + // Set when cert is no longer current; the connection closes once its + // in-flight streams finish. + retired atomic.Bool } func (h *h2ClientConn) isClosed() bool { return h.cc.Err() != nil || h.raw.firstErr() != nil } +func (h *h2ClientConn) retire() { + h.retired.Store(true) + h.cc.SetStateHook(func(cc *http.ClientConn) { + if cc.InFlight() == 0 { + _ = cc.Close() + } + }) + if h.cc.InFlight() == 0 { + _ = h.cc.Close() + } +} + func (h *h2ClientConn) connErr(fallback error) error { if err := h.raw.firstErr(); err != nil { return err diff --git a/internal/atunnel/egress/client_test.go b/internal/atunnel/egress/client_test.go index ee164a6d17..3e2e95e010 100644 --- a/internal/atunnel/egress/client_test.go +++ b/internal/atunnel/egress/client_test.go @@ -309,6 +309,97 @@ func TestClientDialContextHTTP2ReconnectsAfterConnectionClose(t *testing.T) { } } +func TestClientDialContextHTTP2RetiresConnectionAfterRenewal(t *testing.T) { + ca := newTestCA(t) + serials := make(chan string, 4) + gatewayAddress, _ := serveTestH2ConnectGateway(t, ca, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + serials <- r.TLS.PeerCertificates[0].SerialNumber.String() + w.WriteHeader(http.StatusOK) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + _, _ = io.Copy(w, r.Body) + })) + + issue := func() *tls.Certificate { + cert := ca.issue(t, + "spiffe://substrate-actor.local/worker-for-actor/team/actor", + []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + ) + return &cert + } + var current atomic.Pointer[tls.Certificate] + current.Store(issue()) + trustPath := filepath.Join(t.TempDir(), "trust.pem") + if err := os.WriteFile(trustPath, ca.certPEM, 0o600); err != nil { + t.Fatal(err) + } + firstConnClosed := make(chan struct{}) + var tcpDials atomic.Int32 + client, err := NewClient(ClientConfig{ + GatewayAddress: "127.0.0.1:1", + ServerName: "egress.test", + GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return current.Load(), nil }, + TrustBundlePath: trustPath, + }, withDialer(func(ctx context.Context, network, _ string) (net.Conn, error) { + c, err := (&net.Dialer{}).DialContext(ctx, network, gatewayAddress) + if err != nil { + return nil, err + } + if tcpDials.Add(1) == 1 { + return &closeNotifyingConn{Conn: c, closed: firstConnClosed}, nil + } + return c, nil + })) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = client.Close() }) + + before, err := client.DialContext(context.Background(), "192.0.2.10:443") + if err != nil { + t.Fatal(err) + } + defer before.Close() + first := receiveWithin(t, serials, "CONNECT before the renewal") + + current.Store(issue()) + + after, err := client.DialContext(context.Background(), "192.0.2.11:443") + if err != nil { + t.Fatal(err) + } + defer after.Close() + if second := receiveWithin(t, serials, "CONNECT after the renewal"); second == first { + t.Errorf("CONNECT after the renewal presented certificate %s, want the renewed one", second) + } + if dials := tcpDials.Load(); dials != 2 { + t.Errorf("tcpDials = %d, want 2", dials) + } + select { + case <-firstConnClosed: + t.Fatal("first connection closed while its stream was still open") + default: + } + + if _, err := io.WriteString(before, "still-open"); err != nil { + t.Fatal(err) + } + if cw, ok := before.(interface{ CloseWrite() error }); ok { + _ = cw.CloseWrite() + } + got, err := io.ReadAll(before) + if err != nil { + t.Fatal(err) + } + if string(got) != "still-open" { + t.Errorf("stream opened before the renewal got %q, want still-open", got) + } + _ = before.Close() + + receiveWithin(t, firstConnClosed, "retired connection to close after its last stream finished") +} + func TestClientDialContextHTTP2Deadlines(t *testing.T) { ca := newTestCA(t) releaseServerSend := make(chan struct{}) From 4150e4bcd35d46d667ac36b3bf8b72a86ece9867 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 27 Oct 2026 17:50:25 +0000 Subject: [PATCH 14/53] microvm: forward sysctls and lift the guest's privileged-port limit to match gVisor (#1904) A micro-VM container that drops `NET_BIND_SERVICE` could not bind a port below 1024; the same container on gVisor could, since its network stack has no privileged-port limit. The kata agent now receives the spec's sysctls, and ShapeMicroVM sets `net.ipv4.ip_unprivileged_port_start=0`, so both runtimes behave the same. Neither sets `no_new_privileges`; gVisor runs with `--allow-suid` since #2361, so setuid binaries work on both. The capabilities e2e suite gains a drop-ALL template whose probe listens on port 80, and the probe reports `NoNewPrivs`, asserted off on both lanes. - [x] Tests pass: `go test -race ./internal/ocispec/...`, `GOOS=linux go vet` on kata and ocispec, `go vet ./internal/e2e/...`; e2e on both lanes in CI. - [x] Appropriate changes to documentation are included in the PR: docs/api-guide.md. --------- Signed-off-by: Davanum Srinivas --- .../internal/kata/specconv.go | 3 +- .../internal/kata/specconv_linux_test.go | 12 ++++++ docs/api-guide.md | 2 + .../capabilities-templates.yaml.tmpl | 38 +++++++++++++++---- internal/e2e/fixtures/probe/main.go | 5 +++ internal/e2e/sandbox_test.go | 2 +- .../suites/capabilities/capabilities_test.go | 16 +++++++- internal/ocispec/microvm.go | 13 +++++-- internal/ocispec/microvm_test.go | 24 ++++++++++++ 9 files changed, 101 insertions(+), 14 deletions(-) diff --git a/cmd/ateworker-microvm/internal/kata/specconv.go b/cmd/ateworker-microvm/internal/kata/specconv.go index 5f143bb8a6..891bf07878 100644 --- a/cmd/ateworker-microvm/internal/kata/specconv.go +++ b/cmd/ateworker-microvm/internal/kata/specconv.go @@ -93,9 +93,10 @@ func SpecToAgentPB(s *specs.Spec) *agentpb.Spec { CgroupsPath: s.Linux.CgroupsPath, MaskedPaths: s.Linux.MaskedPaths, ReadonlyPaths: s.Linux.ReadonlyPaths, + Sysctl: s.Linux.Sysctl, } // TODO: forward the remaining OCI security knobs the kata-agent supports - // for parity with the OCI spec — Linux.Seccomp and Linux.Sysctl here, and + // for parity with the OCI spec — Linux.Seccomp here, and // Process.ApparmorProfile / Process.SelinuxLabel above. The MVP runs the // actor with kata's defaults for these. for _, ns := range s.Linux.Namespaces { diff --git a/cmd/ateworker-microvm/internal/kata/specconv_linux_test.go b/cmd/ateworker-microvm/internal/kata/specconv_linux_test.go index 71d321c248..4b7d3afdc7 100644 --- a/cmd/ateworker-microvm/internal/kata/specconv_linux_test.go +++ b/cmd/ateworker-microvm/internal/kata/specconv_linux_test.go @@ -162,3 +162,15 @@ func TestSpecToAgentPB_NonPositiveCPUValuesAreDropped(t *testing.T) { }) } } + +// The agent applies sysctls from the spec it is sent; a dropped map leaves +// the guest at its defaults. +func TestSpecToAgentPB_ForwardsSysctl(t *testing.T) { + const key = "net.ipv4.ip_unprivileged_port_start" + got := SpecToAgentPB(&specs.Spec{ + Linux: &specs.Linux{Sysctl: map[string]string{key: "0"}}, + }) + if got.Linux == nil || got.Linux.Sysctl[key] != "0" { + t.Fatalf("Linux.Sysctl = %v, want %s=0 carried through", got.GetLinux().GetSysctl(), key) + } +} diff --git a/docs/api-guide.md b/docs/api-guide.md index a209570dd6..e72afcdf19 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -303,6 +303,8 @@ Each container runs with a default set of Linux capabilities — `AUDIT_WRITE`, - **`ALL` in `add` is rejected.** Kubernetes accepts it in the API and relies on PodSecurity admission to deny it; Substrate has no equivalent policy layer yet, so it is refused at admission instead. Name the capabilities the container needs. - **Ambient capabilities are not supported** ([gvisor#3166](https://github.com/google/gvisor/issues/3166)). +- **Low ports.** A container may bind a port below 1024 without `NET_BIND_SERVICE` on both runtimes: gVisor's network stack has no privileged-port limit, and the micro-VM guest sets `net.ipv4.ip_unprivileged_port_start=0` to match. +- **Setuid binaries work.** Neither runtime sets `no_new_privileges`, as for a Kubernetes container with `allowPrivilegeEscalation: true`; gVisor runs with `--allow-suid`. The sandbox — gVisor or microVM — remains the isolation boundary; capabilities constrain the workload *inside* it. diff --git a/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl b/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl index 50058605f2..549ec8cdc9 100644 --- a/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl +++ b/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl @@ -14,7 +14,7 @@ # The capabilities fixture's ActorTemplates: protojson-shaped # ateapi.ActorTemplate documents (NOT the ate.dev/v1 CRD), created -# through the ate API by e2e.DeploySubstrateFixture. Two templates run the +# through the ate API by e2e.DeploySubstrateFixture. Three templates run the # same probe image and differ only in securityContext.capabilities, so the # suite can compare what the kernel reports inside the sandbox against what # the template asked for. @@ -45,9 +45,8 @@ snapshotConfig: --- -# drop ALL + add: the resulting set is exact rather than relative, so asserting -# it proves both halves at once — everything default was dropped, and only the -# named capability was granted back. +# The set is exact rather than relative, so asserting it proves the drop and +# the add at once. metadata: atespace: ate-e2e${FIXTURE_SUFFIX} name: caps-exact @@ -58,9 +57,6 @@ containers: - name: probe image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe command: ["/ko-app/probe"] - # The probe binds :80, which is exactly what NET_BIND_SERVICE permits, so - # this template also proves the granted capability is usable and not merely - # present in the mask: without it the container could not become ready. securityContext: capabilities: drop: ["ALL"] @@ -76,3 +72,31 @@ resources: memoryBytes: 536870912 # 512Mi snapshotConfig: storageLocation: gs://${BUCKET_NAME}/ate-e2e-caps-exact${FIXTURE_SUFFIX}/ +--- + +# No capabilities at all, and the probe still binds :80: both runtimes lift +# the privileged-port limit, so the golden bake fails if that regresses. +metadata: + atespace: ate-e2e${FIXTURE_SUFFIX} + name: caps-none +workerSelector: + matchLabels: + workload: caps +containers: +- name: probe + image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe + command: ["/ko-app/probe"] + securityContext: + capabilities: + drop: ["ALL"] + wakeupProbe: + httpGet: + path: /healthz + port: 80 + timeoutSeconds: 60 +resources: + limits: + cpuMillicores: 1000 + memoryBytes: 536870912 # 512Mi +snapshotConfig: + storageLocation: gs://${BUCKET_NAME}/ate-e2e-caps-none${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/probe/main.go b/internal/e2e/fixtures/probe/main.go index 2d0cc90888..2deb850071 100644 --- a/internal/e2e/fixtures/probe/main.go +++ b/internal/e2e/fixtures/probe/main.go @@ -80,6 +80,7 @@ type capabilitiesResponse struct { Permitted []string `json:"permitted"` Inheritable []string `json:"inheritable"` Ambient []string `json:"ambient"` + NoNewPrivs bool `json:"noNewPrivs"` // Error carries a read/parse failure so a failing assertion explains itself // instead of just showing empty sets. Error string `json:"error,omitempty"` @@ -130,6 +131,10 @@ func readProcCapabilities() (*capabilitiesResponse, error) { if !found { continue } + if key == "NoNewPrivs:" { + out.NoNewPrivs = strings.TrimSpace(value) == "1" + continue + } dst, ok := targets[strings.TrimSpace(key)] if !ok { continue diff --git a/internal/e2e/sandbox_test.go b/internal/e2e/sandbox_test.go index 423c4618e4..2b4a7db7d6 100644 --- a/internal/e2e/sandbox_test.go +++ b/internal/e2e/sandbox_test.go @@ -58,7 +58,7 @@ var substrateFixtures = []struct { {SubstrateFixtureManifests{ Pool: "internal/e2e/fixtures/capabilities/capabilities.yaml.tmpl", Template: "internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl", - }, 2, nil}, + }, 3, nil}, {SubstrateFixtureManifests{ Pool: "internal/e2e/fixtures/testserver/websocket.yaml.tmpl", Template: "internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl", diff --git a/internal/e2e/suites/capabilities/capabilities_test.go b/internal/e2e/suites/capabilities/capabilities_test.go index 11b32bcbd6..0f03febd7f 100644 --- a/internal/e2e/suites/capabilities/capabilities_test.go +++ b/internal/e2e/suites/capabilities/capabilities_test.go @@ -39,6 +39,7 @@ type capabilitiesResponse struct { Permitted []string `json:"permitted"` Inheritable []string `json:"inheritable"` Ambient []string `json:"ambient"` + NoNewPrivs bool `json:"noNewPrivs"` Error string `json:"error"` } @@ -76,6 +77,13 @@ func TestActorCapabilities(t *testing.T) { name: "drop ALL plus add yields exactly the added capability", template: "caps-exact", want: []string{"NET_BIND_SERVICE"}, + }, { + // The golden bake in deployFixture is what proves port 80 binds without + // NET_BIND_SERVICE; this row checks the restored actor still answers + // with an empty set. + name: "drop ALL still binds port 80", + template: "caps-none", + want: nil, }} for _, tt := range tests { @@ -110,6 +118,10 @@ func TestActorCapabilities(t *testing.T) { if len(got.Ambient) != 0 { t.Errorf("ambient = %v, want empty (ambient capabilities are not supported)", got.Ambient) } + // Neither runtime sets no_new_privileges; gVisor runs with --allow-suid. + if got.NoNewPrivs { + t.Error("no_new_privs = 1, want 0") + } }) } } @@ -128,8 +140,8 @@ func assertSameCapabilities(t *testing.T, set string, got, want []string) { // deployFixture installs the fixture for the sandbox class under test and // returns its atespace (which also names the namespace it created, carrying -// the class suffix so the gVisor and micro-VM lanes never share one). Both -// templates are golden-snapshotted when this returns; a template whose +// the class suffix so the gVisor and micro-VM lanes never share one). Every +// template is golden-snapshotted when this returns; a template whose // container cannot start — for example because a needed capability was // dropped — fails the deploy with the template's error message rather than // timing out per subtest. diff --git a/internal/ocispec/microvm.go b/internal/ocispec/microvm.go index d446cc359b..c0eff0e143 100644 --- a/internal/ocispec/microvm.go +++ b/internal/ocispec/microvm.go @@ -57,9 +57,9 @@ type MicroVMOptions struct { } // ShapeMicroVM replaces host system mounts with guest mounts, repoints volume -// bind mounts to guest share paths, and fills in kata's default resources. It -// must run on an unshaped spec, and errors on a bind it cannot place in the -// guest. +// bind mounts to guest share paths, fills in kata's default resources, and +// lifts the guest's privileged-port limit. It must run on an unshaped spec, +// and errors on a bind it cannot place in the guest. func ShapeMicroVM(spec *specs.Spec, o MicroVMOptions) error { // Translate volume bind mounts into guest share paths. volumes := make([]specs.Mount, 0, len(spec.Mounts)) @@ -86,6 +86,13 @@ func ShapeMicroVM(spec *specs.Spec, o MicroVMOptions) error { if spec.Linux == nil { spec.Linux = &specs.Linux{} } + // gVisor has no privileged-port limit, and runsc ignores this key. + if spec.Linux.Sysctl == nil { + spec.Linux.Sysctl = map[string]string{} + } + if _, ok := spec.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"]; !ok { + spec.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"] = "0" + } // The container's own declared limits survive the merge; StartRootfsContainer // sets CgroupsPath. spec.Linux.Resources = mergeKataResources(spec.Linux.Resources) diff --git a/internal/ocispec/microvm_test.go b/internal/ocispec/microvm_test.go index 74bdbb8b08..9e376388f2 100644 --- a/internal/ocispec/microvm_test.go +++ b/internal/ocispec/microvm_test.go @@ -171,3 +171,27 @@ func TestShapeMicroVM_LeavesUndeclaredContainerUnlimited(t *testing.T) { t.Errorf("cpu quota = %d, want unset for a container that declared none", *c.Quota) } } + +func TestShapeMicroVM_AllowsLowPortsInTheGuest(t *testing.T) { + const portStart = "net.ipv4.ip_unprivileged_port_start" + + spec := Build(Options{Container: &ateworkerpb.Container{ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app"}}}}) + if _, ok := spec.Linux.Sysctl[portStart]; ok { + t.Errorf("Build() set %s; it must be micro-VM only", portStart) + } + if err := ShapeMicroVM(spec, MicroVMOptions{ActorDirs: parityActorDirs, ContainerName: "app"}); err != nil { + t.Fatalf("ShapeMicroVM() = %v", err) + } + if v := spec.Linux.Sysctl[portStart]; v != "0" { + t.Errorf("%s = %q, want \"0\"", portStart, v) + } + + preset := Build(Options{Container: &ateworkerpb.Container{ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app"}}}}) + preset.Linux.Sysctl = map[string]string{portStart: "1024"} + if err := ShapeMicroVM(preset, MicroVMOptions{ActorDirs: parityActorDirs, ContainerName: "app"}); err != nil { + t.Fatalf("ShapeMicroVM() = %v", err) + } + if v := preset.Linux.Sysctl[portStart]; v != "1024" { + t.Errorf("%s = %q, want the caller's value kept", portStart, v) + } +} From 6a67f431e0f2d407eaaa73fa03756fac8a07ffcb Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Tue, 27 Oct 2026 18:16:04 +0000 Subject: [PATCH 15/53] Update agentgateway version to fix drift and CI (#2962) Why is this needed? (Or link the issue. Open one first for features, API, install or CI changes.) Fixes the agentgateway postsubmit by updating the agentgateway implementation after some substrate contract changes. As part of this, I expanded some tests to be less Envoy-specific in their assertions. AI use: AI assisted; I reviewed and ran local e2e tests - [X] Tests pass - [X] Docs updated --------- Signed-off-by: Keith Mattix II Signed-off-by: Eitan Yarmush Co-authored-by: Eitan Yarmush --- cmd/ate-setup/commands.md | 2 +- cmd/ate-setup/internal/config/config.go | 11 +- cmd/ate-setup/internal/config/config_test.go | 6 +- cmd/ate-setup/internal/config/setting.go | 2 +- cmd/ate-setup/internal/steps/overlay.go | 42 +++- cmd/ate-setup/internal/steps/overlay_test.go | 167 +++++++++++++++- docs/architecture.md | 11 +- docs/dev/mitm-egress-test-runbook.md | 8 + docs/egress-traffic.md | 4 + docs/network-egress.md | 35 +++- internal/e2e/atenet_dataplane.go | 19 +- internal/e2e/atenet_dataplane_test.go | 21 ++ internal/e2e/fixtures/probe/main.go | 22 +++ internal/e2e/fixtures/probe/main_test.go | 29 +++ .../e2e/suites/egressmitm/egressmitm_test.go | 30 +-- .../suites/ingressauth/ingressauth_test.go | 183 +++++++++++++++--- .../suites/networking/egresspolicy_test.go | 7 +- .../e2e/suites/networking/networking_test.go | 6 +- .../agentgateway-egress/kustomization.yaml | 4 +- .../agentgateway-static-mtls/configmap.yaml | 120 ++++++++++++ .../kustomization.yaml | 24 +++ .../components/agentgateway/configmap.yaml | 14 +- .../agentgateway/kustomization.yaml | 4 +- 23 files changed, 683 insertions(+), 88 deletions(-) create mode 100644 manifests/ate-install/components/agentgateway-static-mtls/configmap.yaml create mode 100644 manifests/ate-install/components/agentgateway-static-mtls/kustomization.yaml diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index c47ce3da89..a092188c92 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -31,7 +31,7 @@ a pre-scan pass, so they may appear anywhere on its command line. | `--ateapi-postgres-storage-class NAME` | `ATE_API_POSTGRES_STORAGE_CLASS=NAME` | StorageClass for the bundled PostgreSQL volume instead of the cluster default, such as `dynamic-rwo` on GKE so C4 and N4 nodes can attach it. An existing install must delete the postgres StatefulSet and its claim first | | `--cordon-control-plane` | `--cordon-control-plane` | Keep the control plane off the worker nodes. Assumes a small shared pool labeled and tainted `ate.dev/workloadType=ate-control-plane:NoSchedule`, across which each workload's replicas are spread, and a one-node pool labeled and tainted `ate.dev/workloadType=ate-postgres:NoSchedule` for postgres alone. `ATE_INSTALL_CORDON_CONTROL_PLANE=true` when the flag is absent | | `--fake-data-plane` | `--fake-data-plane` | Install the fake data plane for control-plane benchmarks: fake-atelet takes atelet's place in its DaemonSet, taking its objectstore-plugin's storage settings, ate-controller is scaled to zero, and fake-workersync honors every WorkerPool with fake Workers on the fake-atelet nodes. Reaches `deploy ate-system`, `deploy atelet`, and `deploy ate-controller`; only `deploy ate-system` switches a cluster between the real and fake data planes, and the other two refuse a data plane other than the installed one. `deploy ate-system --fake-data-plane` refuses a cluster whose WorkerPools have worker pods. fake-atelet runs with its defaults; a benchmark tunes it through the optional `fake-atelet-tuning` ConfigMap, whose `FAKE_ATELET_*` variables set its flags, and which no install creates or resets. Actors do not exist; never use it on a cluster that serves real actors. `ATE_INSTALL_FAKE_DATA_PLANE=true` when the flag is absent | -| `--ingress-allowed-spiffe-ids IDS` | `--ingress-allowed-spiffe-ids IDS`, or `ATE_INGRESS_ALLOWED_SPIFFE_IDS=IDS` | Required by `deploy ate-system` and `deploy atenet` with `--atenet-dataplane=envoy`. Comma-separated SPIFFE IDs (exact match) of the clients the ingress router lets reach actors over its TLS ports; any listed client may reach any actor. Appended to the router as its `--ingress-allowed-spiffe-ids`. The podidentity signer issues `spiffe://cluster.local/ns//sa/` | +| `--ingress-allowed-spiffe-ids IDS` | `--ingress-allowed-spiffe-ids IDS`, or `ATE_INGRESS_ALLOWED_SPIFFE_IDS=IDS` | Required by `deploy ate-system` and `deploy atenet` for both dataplanes. Comma-separated SPIFFE IDs (exact match) of the clients the ingress router lets reach actors; any listed client may reach any actor. Both dataplanes require a Pod Identity CA client certificate on HTTPS and TLS CONNECT. The SPIFFE URI SAN allowlist may be checked during or after TLS, including after CONNECT acceptance, but unlisted clients must be denied before actor routing. The podidentity signer issues `spiffe://cluster.local/ns//sa/` | | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | | `--credential-provider JSON` | `--credential-provider JSON`, or `ATE_CREDENTIAL_PROVIDER` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/egress-credential-injection.md`](../../docs/egress-credential-injection.md) | | `--ateapi-log-level LEVEL` | `--ateapi-log-level LEVEL`, or `ATE_API_LOG_LEVEL=LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). The server picks up a change without restarting | diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index ab39897cf8..68d8c6a6fa 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -276,13 +276,12 @@ func (c *Config) Resolved() *Resolved { return c.resolved } -// RequireIngressAllowedSPIFFEIDs fails when the install would deploy the -// Envoy ingress router without an allowlist. That router authenticates -// clients on its TLS listeners, and refuses to start with no SPIFFE ID to -// match them against. The agentgateway router is configured statically and -// takes none. +// RequireIngressAllowedSPIFFEIDs fails when the install would deploy an +// ingress router without an allowlist. Envoy and +// AgentGateway both authenticate clients on their TLS listeners and use the +// allowlist to decide which authenticated identities may reach actors. func (c *Config) RequireIngressAllowedSPIFFEIDs() error { - if c.Router == RouterAgentgateway || len(c.IngressAllowedSPIFFEIDs) > 0 { + if len(c.IngressAllowedSPIFFEIDs) > 0 { return nil } return c.Resolved().Require("ingressAllowedSPIFFEIDs") diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 99ae080bbd..b25ca310c7 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -287,8 +287,7 @@ func TestLoadIngressAllowedSPIFFEIDs(t *testing.T) { } } -// Only the Envoy router authenticates clients against the allowlist, and it -// refuses to start without one. +// Both ingress dataplanes require an allowlist. func TestRequireIngressAllowedSPIFFEIDs(t *testing.T) { for _, tc := range []struct { name string @@ -297,7 +296,8 @@ func TestRequireIngressAllowedSPIFFEIDs(t *testing.T) { }{ {name: "envoy without IDs", cfg: Config{Router: RouterEnvoy}, wantErr: true}, {name: "envoy with IDs", cfg: Config{Router: RouterEnvoy, IngressAllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/a/sa/a"}}}, - {name: "agentgateway without IDs", cfg: Config{Router: RouterAgentgateway}}, + {name: "agentgateway without IDs", cfg: Config{Router: RouterAgentgateway}, wantErr: true}, + {name: "agentgateway with IDs", cfg: Config{Router: RouterAgentgateway, IngressAllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/a/sa/a"}}}, } { t.Run(tc.name, func(t *testing.T) { err := tc.cfg.RequireIngressAllowedSPIFFEIDs() diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 389da4d55c..97b67e8f11 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -307,7 +307,7 @@ var Registry = []Setting{ Kind: KindString, Usage: "SPIFFE IDs (exact match, comma-separated) of the clients the ingress router lets reach actors; " + "any listed client may reach any actor. Required to deploy the router with " + - "--atenet-dataplane=envoy. The podidentity signer issues spiffe://cluster.local/ns//sa/", + "either dataplane. The podidentity signer issues spiffe://cluster.local/ns//sa/", }, { Key: "otlpEndpoint", Env: "ATE_OTLP_ENDPOINT", Flag: "otlp-endpoint", Kind: KindString, diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 8a2d6d3e36..89d82ca4f3 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -17,6 +17,7 @@ package steps import ( "bytes" "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -48,6 +49,8 @@ const cordonControlPlaneComponent = installDir + "/components/cordon-control-pla // over the atelet and ate-controller applies under --fake-data-plane. const fakeDataPlaneComponent = installDir + "/components/fake-data-plane" +const agentgatewayStaticMTLSComponent = installDir + "/components/agentgateway-static-mtls" + // components lists the kustomize components every control plane apply is // composed with. Each patch targets its workloads by name, and kustomize leaves // a stream alone when nothing in it matches, so composing a manifest that @@ -57,6 +60,9 @@ func (e *Env) components() []string { if e.Cfg.FakeDataPlane { components = append(components, e.Cfg.Path(fakeDataPlaneComponent)) } + if e.Cfg.Router == config.RouterAgentgateway { + components = append(components, e.Cfg.Path(agentgatewayStaticMTLSComponent)) + } if e.Cfg.CordonControlPlane { components = append(components, e.Cfg.Path(cordonControlPlaneComponent)) } @@ -142,6 +148,9 @@ func (e *Env) renderSystemManifests(ctx context.Context) ([]byte, error) { if manifest, err = setIDPServerIssuer(manifest, e.Cfg.ActorJWTIssuer); err != nil { return nil, err } + if manifest, err = e.setAgentgatewayIngressAllowedSPIFFEIDs(manifest); err != nil { + return nil, err + } if e.Cfg.Router != config.RouterAgentgateway { manifest, err = e.patchAtenetRouterAllowedSPIFFEIDs(manifest) if err != nil { @@ -160,7 +169,15 @@ func (e *Env) renderSystemManifests(ctx context.Context) ([]byte, error) { // selected dataplane. func (e *Env) renderAtenetRouterManifest(ctx context.Context) ([]byte, error) { if e.Cfg.Router == config.RouterAgentgateway { - return e.renderResolve(ctx, e.Cfg.Path(installDir+"/agentgateway-router")) + manifest, err := e.render(e.Cfg.Path(installDir + "/agentgateway-router")) + if err != nil { + return nil, err + } + manifest, err = e.setAgentgatewayIngressAllowedSPIFFEIDs(manifest) + if err != nil { + return nil, err + } + return e.ResolveManifestBytes(ctx, manifest) } manifest, err := e.render(e.Cfg.Manifest("atenet-router.yaml")) if err != nil { @@ -178,6 +195,25 @@ func (e *Env) renderAtenetRouterManifest(ctx context.Context) ([]byte, error) { return e.ResolveManifestBytes(ctx, manifest) } +func (e *Env) setAgentgatewayIngressAllowedSPIFFEIDs(raw []byte) ([]byte, error) { + if e.Cfg.Router != config.RouterAgentgateway { + return raw, nil + } + if err := e.Cfg.RequireIngressAllowedSPIFFEIDs(); err != nil { + return nil, err + } + + ids, err := json.Marshal(e.Cfg.IngressAllowedSPIFFEIDs) + if err != nil { + return nil, fmt.Errorf("encoding AgentGateway ingress SPIFFE IDs: %w", err) + } + marker := []byte("[ATE_INGRESS_ALLOWED_SPIFFE_IDS]") + if count := bytes.Count(raw, marker); count != 1 { + return nil, fmt.Errorf("expected 1 AgentGateway ingress SPIFFE IDs placeholder, found %d", count) + } + return bytes.Replace(raw, marker, ids, 1), nil +} + // atenetEgressManifestPath returns the envoy egress gateway manifest. func (e *Env) atenetEgressManifestPath() string { return e.Cfg.Manifest("atenet-egress.yaml") @@ -198,6 +234,10 @@ func (e *Env) renderAtenetEgressManifest(ctx context.Context, provider config.Cr if err != nil { return nil, err } + raw, err = e.setAgentgatewayIngressAllowedSPIFFEIDs(raw) + if err != nil { + return nil, err + } raw, err = patchAgentgatewayEgressInject(raw, provider) if err != nil { return nil, err diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index a626532e64..814ca752e1 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -830,6 +830,166 @@ func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { } } +func TestRenderAgentgatewayStaticMTLS(t *testing.T) { + const ( + idA = "spiffe://cluster.local/ns/a/sa/a" + idB = "spiffe://cluster.local/ns/b/sa/b" + ) + src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} + resolver := images.NewPrebuilt(src, func(context.Context, string) (string, error) { + return "sha256:" + strings.Repeat("2", 64), nil + }) + for _, tc := range []struct { + name string + render func(*Env) ([]byte, error) + }{ + { + name: "system bundle", + render: func(e *Env) ([]byte, error) { + return e.renderSystemManifests(t.Context()) + }, + }, + { + name: "router redeploy", + render: func(e *Env) ([]byte, error) { + return e.renderAtenetRouterManifest(t.Context()) + }, + }, + { + name: "kind system bundle", + render: func(e *Env) ([]byte, error) { + e.Cfg.Kind = true + return e.renderSystemManifests(t.Context()) + }, + }, + { + name: "egress redeploy", + render: func(e *Env) ([]byte, error) { + return e.renderAtenetEgressManifest(t.Context(), config.CredentialProvider{}) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := &config.Config{ + Root: repoRoot(t), + Router: config.RouterAgentgateway, + IngressAllowedSPIFFEIDs: []string{idA, idB}, + Images: src, + } + e := &Env{Cfg: cfg, resolver: resolver} + manifest, err := tc.render(e) + if err != nil { + t.Fatalf("rendering static-mtls AgentGateway manifests: %v", err) + } + cm := agentgatewayRouterConfig(t, manifest) + var gateway map[string]interface{} + if err := yaml.Unmarshal([]byte(cm.Data["config.yaml"]), &gateway); err != nil { + t.Fatalf("parsing rendered AgentGateway config: %v", err) + } + gateways, ok := gateway["gateways"].(map[string]interface{}) + if !ok { + t.Fatal("rendered AgentGateway config has no gateways mapping") + } + if _, exists := gateways["http"]; exists { + t.Error("static-mtls config still has a plaintext HTTP gateway") + } + https, ok := gateways["https"].(map[string]interface{}) + if !ok { + t.Fatal("static-mtls config has no HTTPS gateway") + } + assertAgentgatewayClientRoot(t, https) + policies := gateway["frontendPolicies"].(map[string]interface{}) + authorization := policies["networkAuthorization"].(map[string]interface{}) + wantRules := []interface{}{ + map[string]interface{}{"allow": fmt.Sprintf("source.spiffeId in [%q,%q]", idA, idB)}, + } + if got := authorization["rules"]; !reflect.DeepEqual(got, wantRules) { + t.Errorf("network authorization rules = %v, want %v", got, wantRules) + } + + binds, ok := gateway["binds"].([]interface{}) + if !ok { + t.Fatal("rendered AgentGateway config has no binds list") + } + var connectTLSListener map[string]interface{} + for _, item := range binds { + bind, ok := item.(map[string]interface{}) + if !ok { + t.Fatalf("invalid bind in rendered AgentGateway config: %T", item) + } + port, ok := bind["port"].(float64) + if !ok { + if _, exists := bind["port"]; exists { + t.Fatalf("invalid bind port in rendered AgentGateway config: %v", bind["port"]) + } + continue + } + if port == 8081 { + t.Error("static-mtls config still has a plaintext CONNECT bind") + } + if port != 8444 { + continue + } + listeners, ok := bind["listeners"].([]interface{}) + if !ok { + t.Fatal("TLS CONNECT bind has no listeners") + } + for _, listenerValue := range listeners { + listener, ok := listenerValue.(map[string]interface{}) + if ok && listener["protocol"] == "HTTPS" { + connectTLSListener = listener + } + } + } + if connectTLSListener == nil { + t.Fatal("static-mtls config has no TLS CONNECT listener") + } + assertAgentgatewayClientRoot(t, connectTLSListener) + + t.Run("without IDs", func(t *testing.T) { + cfg.IngressAllowedSPIFFEIDs = nil + _, err := tc.render(e) + var required *config.RequiredError + if !errors.As(err, &required) { + t.Fatalf("render error = %v, want a RequiredError", err) + } + }) + }) + } +} + +func agentgatewayRouterConfig(t *testing.T, manifest []byte) *corev1.ConfigMap { + t.Helper() + for _, doc := range strings.Split(string(manifest), "\n---\n") { + var cm corev1.ConfigMap + if err := yaml.Unmarshal([]byte(doc), &cm); err != nil { + t.Fatalf("rendered document is not valid YAML: %v", err) + } + if cm.Kind == "ConfigMap" && cm.Name == "atenet-router-agentgateway-config" { + if cm.Data["config.yaml"] == "" { + t.Fatal("AgentGateway router ConfigMap has no config.yaml") + } + return &cm + } + } + t.Fatal("rendered manifest has no AgentGateway router ConfigMap") + return nil +} + +func assertAgentgatewayClientRoot(t *testing.T, config map[string]interface{}) { + t.Helper() + tls, ok := config["tls"].(map[string]interface{}) + if !ok { + t.Fatal("TLS listener has no TLS config") + } + if got := tls["root"]; got != "/run/podidentity.podcert.ate.dev/trust-bundle.pem" { + t.Errorf("TLS client CA = %v, want podidentity trust bundle", got) + } + if _, exists := tls["allowedClientSpiffeIds"]; exists { + t.Error("allowlist must use the existing network authorization policy") + } +} + // routerArgs returns the atenet-router container's args from a rendered // manifest. func routerArgs(t *testing.T, manifest []byte) []string { @@ -953,7 +1113,12 @@ func TestRenderAgentgatewayCredentialProvider(t *testing.T) { t.Run(tc.name, func(t *testing.T) { src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} e := &Env{ - Cfg: &config.Config{Root: repoRoot(t), Router: config.RouterAgentgateway, Images: src}, + Cfg: &config.Config{ + Root: repoRoot(t), + Router: config.RouterAgentgateway, + Images: src, + IngressAllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/client/sa/client"}, + }, resolver: images.NewPrebuilt(src, func(context.Context, string) (string, error) { return "sha256:" + strings.Repeat("2", 64), nil }), diff --git a/docs/architecture.md b/docs/architecture.md index 8336ab2659..98d6644132 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -561,8 +561,15 @@ Agent Substrate is built on a **Defense-in-Depth** model: the actor its `ate-target-actor` header names, and only to recognized, registered actors. It accepts only mTLS clients whose SPIFFE ID is on a static allowlist; any listed client may reach any actor, as per-actor - ingress authorization is not yet supported. The agentgateway dataplane - does not authenticate clients yet. + ingress authorization is not yet supported. + The installer takes that allowlist from `--ingress-allowed-spiffe-ids` + (`ATE_INGRESS_ALLOWED_SPIFFE_IDS` or `ingressAllowedSPIFFEIDs` in its config). + Both dataplanes must verify the client certificate against the Pod Identity + CA during the TLS handshake on both HTTPS and TLS CONNECT listeners. + The SPIFFE URI SAN allowlist may be checked during or after TLS, but an + unlisted client must not reach actor routing. A CONNECT tunnel may be + accepted before that check, but traffic inside it must still be denied. + Neither dataplane serves plaintext ingress listeners. * **Network Policy**: Agent Substrate leverages standard Kubernetes NetworkPolicy for connectivity control. Policies can be applied at the diff --git a/docs/dev/mitm-egress-test-runbook.md b/docs/dev/mitm-egress-test-runbook.md index 789ed24497..2d3a7c978c 100644 --- a/docs/dev/mitm-egress-test-runbook.md +++ b/docs/dev/mitm-egress-test-runbook.md @@ -78,3 +78,11 @@ kubectl --context kind-kind -n ate-system logs deploy/atenet-egress -c envoy --t For AgentGateway, use `-c agentgateway` for the last command. For a local TLS failure, check that `setup-e2e-egress-tls-kind.sh` completed and that the test URL uses the origin's Service DNS name. + +If the AgentGateway ingress router returns HTTP 503 with `dynamic backend +resolved to a loopback or link-local address`, check its +`config.backend.allowDynamicLoopback` setting. The router enables this because +actor-local destinations are reached through authenticated atunnel connections, +not on the router itself. The egress gateway leaves this setting disabled. +Changes to this startup setting require restarting the router; a ConfigMap +reload alone does not apply it. diff --git a/docs/egress-traffic.md b/docs/egress-traffic.md index 0ee6ba5399..224448ccbb 100644 --- a/docs/egress-traffic.md +++ b/docs/egress-traffic.md @@ -26,6 +26,10 @@ cannot reach the collector until the actor has an `http` egress rule for it. See | 53 | DNS to the sandbox's gateway | Allowed for hostnames matching the actor's egress policy | sandbox DNS relay -> worker pod's resolvers | `NXDOMAIN` | | any | Any other TCP | Blocked | n/a | The connection is accepted and then closed with no bytes returned. There is no status code. atunnel logs the failure. | +See the [Network Egress Contract](network-egress.md#refusal-responses-and-policy-changes) +for refusal status codes, policy-refresh semantics, and the distinction between DNS +refusals and HTTP policy denials. + ## UDP diff --git a/docs/network-egress.md b/docs/network-egress.md index 5400c1cede..83be6a6fe5 100644 --- a/docs/network-egress.md +++ b/docs/network-egress.md @@ -1,6 +1,6 @@ # Network Egress Contract -Last updated: 10/09/2026 +Last updated: 10/20/2026 ## Overview @@ -8,10 +8,39 @@ Agent Substrate enforces that all outbound traffic from an actor, except traffic ## Transport -At present, actor TCP traffic exits the sandbox through [`atunnel`](./glossary.md#components) and is forwarded on to the PEP. The exception is DNS: TCP and UDP traffic to port 53 on the sandbox's gateway bypasses the PEP and is answered by the sandbox's DNS relay, which forwards a query to the worker pod's resolvers only when the queried hostname matches a host or host wildcard in the actor's egress policy; queries from an actor without a policy, or for a hostname not matched by any rule, are answered `NXDOMAIN`. All other protocols are blocked. `atunnel` is configured with the PEP's network address and port at startup and uses that address to open an mTLS HTTP/1.1 CONNECT tunnel to the PEP. The CONNECT request target and `Host` header are both set to the original destination IP and port that `atunnel` derived from the actor's TCP connection. Actor networking is currently IPv4-only. +At present, actor TCP traffic exits the sandbox through [`atunnel`](./glossary.md#components) and is forwarded on to the PEP. The exception is DNS: TCP and UDP traffic to port 53 on the sandbox's gateway bypasses the PEP and is answered by the sandbox's DNS relay, which forwards a query to the worker pod's resolvers only when the queried hostname matches a host or host wildcard in the actor's egress policy; queries from an actor without a policy, or for a hostname not matched by any rule, are answered `NXDOMAIN`. All other protocols are blocked. `atunnel` is configured with the PEP's network address and port at startup and uses that address to open an mTLS HTTP/1.1 or HTTP/2 CONNECT tunnel to the PEP, as negotiated by ALPN. The CONNECT request target and HTTP/1.1 `Host` header (HTTP/2 `:authority`) are set to the original destination IP and port that `atunnel` derived from the actor's TCP connection. Actor networking is currently IPv4-only. Before traffic reaches the egress PEP, ateworker rejects packets destined for the IPv4 link-local range `169.254.0.0/16` or the IPv6 link-local range `fe80::/10`. The internal actor network `169.254.17.0/30` is exempt because it carries traffic for the actor gateway, DNS relay, ingress, and wakeup probes. Localhost traffic remains inside the actor sandbox and does not traverse the PEP. Access to non-link-local destinations remains controlled by substrate egress policy and, at the `WorkerPool` boundary, Kubernetes NetworkPolicy. +## Refusal responses and policy changes + +The [supported traffic table](egress-traffic.md) defines the permitted protocols. +For cleartext HTTP requests and HTTP requests on an authorized intercepted TLS +connection, a request denied by policy MUST receive HTTP 403. WebSocket upgrades, +other HTTP upgrades, and actor-originated CONNECT requests inside the tunnel MUST +also receive HTTP 403, regardless of whether the destination is otherwise allowed. +This applies to cleartext HTTP and intercepted HTTPS, including mixed-case HTTP +upgrade tokens, and does not affect atunnel's outer CONNECT. The PEP MUST NOT open an upstream connection +or apply credential effects for these refused requests. Response bodies are +implementation-specific. + +A TLS passthrough authorization lasts for that connection. A policy update MUST +apply to new connections after cached policy decisions expire; it need not +terminate an already authorized connection. If a new ClientHello no longer +matches a passthrough rule, the PEP MUST NOT pass the TLS connection through. +If an HTTPS rule matches the SNI and original destination port, the PEP MAY +intercept the connection and authorize each HTTP request under that rule. +Otherwise, the PEP MUST close the connection at ClientHello, without completing +the TLS handshake, returning an HTTP response, or opening an upstream connection. +An HTTP rule alone MUST NOT authorize TLS. +Missing or invalid SNI MUST also close the connection without completing TLS. +Unsupported protocols MAY close the connection without an HTTP response. + +DNS authorization happens before the PEP and is independent of the protocol and +port being dialed. Removing every rule for a hostname can prevent resolution +before the connection reaches the PEP; that refusal does not promise an HTTP +status or a particular resolver error string. + ## Trust Boundary In substrate, we consider everything originating from the actor to be untrusted. The egress PEP MUST NOT trust any decision or attestation coming from the actor itself; it MUST instead trust only information carried through substrate-controlled channels (e.g. the actor certificate and the CONNECT request produced by atunnel). For example, the PEP MUST NOT treat an actor-provided HTTP hostname or TLS SNI as proof that the original destination has that name. The PEP MAY use the hostname as a policy input, but when authorization is based on that hostname, it MUST route the traffic to the authorized hostname rather than use the hostname to authorize an arbitrary IP address from the CONNECT request. @@ -20,8 +49,6 @@ In substrate, we consider everything originating from the actor to be untrusted. The egress PEP MUST serve TLS using a certificate that is valid for the configured PEP hostname and chains to a CA trusted by `atunnel`. `atunnel` uses the configured PEP hostname as the TLS server name and MUST reject the connection if it cannot verify the PEP's certificate. -In the default Kubernetes deployment, the PEP obtains its certificate and private key through a projected `podCertificate` volume using substrate's Service DNS signer (`servicedns.podcert.ate.dev/identity`). The signer derives the certificate's DNS SANs from the Services that select the PEP Pod, so the PEP MUST be selected by a Service and the hostname configured in `atunnel` MUST match one of those Service DNS names. For the default PEP, that name is `atenet-egress.ate-system.svc`. The projected `credential-bundle.pem` contains the serving certificate chain and private key, while the `ClusterTrustBundle` for the same signer supplies `atunnel` with the corresponding trust anchors. - ## Client Authorization `atunnel` MUST present an actor-specific client certificate when connecting to the PEP. Before the actor's workload starts, `atunnel` generates a private key and requests a short-lived certificate from substrate through the node-local atelet, and egress is active from then on: a workload may need the network to become ready, so egress does not wait for the wakeup probe the way ingress does. The private key remains in `atunnel`. The certificate identifies the actor by its atespace, name, and UID and is scoped to the `atunnel` purpose. diff --git a/internal/e2e/atenet_dataplane.go b/internal/e2e/atenet_dataplane.go index a1066bb947..5267897a1f 100644 --- a/internal/e2e/atenet_dataplane.go +++ b/internal/e2e/atenet_dataplane.go @@ -39,9 +39,6 @@ type AtenetDataplane interface { RouteDurationSeen(context.Context, string) (bool, error) ScrapeMetrics(context.Context) (string, error) EgressConnectStats(before, after string) EgressConnectStats - // AuthenticatesIngressClients reports whether the ingress dataplane - // requires an allowlisted client certificate. - AuthenticatesIngressClients() bool } // EgressConnectStats holds the delta of egress CONNECT connections and requests @@ -80,14 +77,16 @@ func (envoyAtenetDataplane) IsRetryableParkingBudgetExhaustion(status int, body func (envoyAtenetDataplane) ParkingBudgetStatus() int { return http.StatusServiceUnavailable } func (envoyAtenetDataplane) IsEgressPolicyDenied(status int, body string) bool { - return (status == http.StatusForbidden && strings.Contains(body, "egress denied")) || + return isEgressPolicyDenied(status, body) +} + +func isEgressPolicyDenied(status int, body string) bool { + return (status == http.StatusForbidden && (strings.Contains(body, "egress denied") || strings.Contains(body, "actor egress policy denied"))) || (status == http.StatusBadGateway && strings.Contains(body, "request failed")) } func (envoyAtenetDataplane) PlatformMetricPrefixes(prefixes []string) []string { return prefixes } -func (envoyAtenetDataplane) AuthenticatesIngressClients() bool { return true } - func (envoyAtenetDataplane) RouteDurationSeen(_ context.Context, collectorScrape string) (bool, error) { return len(MissingPlatformMetrics(collectorScrape, []string{"atenet_router_route_duration"})) == 0, nil } @@ -141,15 +140,13 @@ func (agentGatewayAtenetDataplane) NewParkingObserver(context.Context) (ParkingO } func (agentGatewayAtenetDataplane) IsRetryableParkingBudgetExhaustion(status int, body string) bool { - return status == http.StatusGatewayTimeout && strings.Contains(body, "request timed out") + return status == http.StatusServiceUnavailable && strings.Contains(body, "no worker has room for the actor") } -func (agentGatewayAtenetDataplane) ParkingBudgetStatus() int { return http.StatusGatewayTimeout } - -func (agentGatewayAtenetDataplane) AuthenticatesIngressClients() bool { return false } +func (agentGatewayAtenetDataplane) ParkingBudgetStatus() int { return http.StatusServiceUnavailable } func (agentGatewayAtenetDataplane) IsEgressPolicyDenied(status int, body string) bool { - return status == http.StatusForbidden && strings.Contains(body, "actor egress policy denied") + return isEgressPolicyDenied(status, body) } func (agentGatewayAtenetDataplane) PlatformMetricPrefixes(prefixes []string) []string { diff --git a/internal/e2e/atenet_dataplane_test.go b/internal/e2e/atenet_dataplane_test.go index 9d3b7c6d45..9a55b82f49 100644 --- a/internal/e2e/atenet_dataplane_test.go +++ b/internal/e2e/atenet_dataplane_test.go @@ -31,9 +31,30 @@ func TestAtenetDataplaneEgressPolicyDenial(t *testing.T) { if !CurrentAtenetDataplane().IsEgressPolicyDenied(http.StatusForbidden, "actor egress policy denied destination") { t.Error("AgentGateway direct policy denial was not recognized") } + if !CurrentAtenetDataplane().IsEgressPolicyDenied(http.StatusBadGateway, `{"error":"request failed: EOF"}`) { + t.Error("AgentGateway ClientHello refusal was not recognized") + } }) } +func TestAtenetDataplaneParkingBudget(t *testing.T) { + for _, dataplane := range []string{"envoy", "agentgateway"} { + t.Run(dataplane, func(t *testing.T) { + t.Setenv(AtenetDataplaneEnv, dataplane) + d := CurrentAtenetDataplane() + if got := d.ParkingBudgetStatus(); got != http.StatusServiceUnavailable { + t.Errorf("parking budget status = %d, want 503", got) + } + if !d.IsRetryableParkingBudgetExhaustion(http.StatusServiceUnavailable, "no worker has room for the actor") { + t.Error("parking budget exhaustion was not recognized") + } + if d.IsRetryableParkingBudgetExhaustion(http.StatusGatewayTimeout, "request timed out") { + t.Error("transport timeout was recognized as parking budget exhaustion") + } + }) + } +} + func TestAtenetDataplaneEgressConnectStats(t *testing.T) { const before = `# TYPE envoy_http_downstream_cx_http2_total counter envoy_http_downstream_cx_http2_total{envoy_http_conn_manager_prefix="egress_connect"} 1 diff --git a/internal/e2e/fixtures/probe/main.go b/internal/e2e/fixtures/probe/main.go index 2deb850071..5dedcb1005 100644 --- a/internal/e2e/fixtures/probe/main.go +++ b/internal/e2e/fixtures/probe/main.go @@ -26,16 +26,20 @@ import ( "crypto/tls" "crypto/x509" "encoding/json" + "errors" "fmt" "io" "log" "net" "net/http" + "net/http/httptrace" "os" "runtime" "strconv" "strings" "sync" + "sync/atomic" + "syscall" "time" ) @@ -355,6 +359,7 @@ func parseFetchHeaders(params []string) (http.Header, error) { // injected credential echoed back by a headers-echo endpoint). Failures land // in "error" rather than the HTTP status: a TLS verification failure is a // result for the suite to assert on, not a broken probe. +// "tls_handshake" reports "complete", "closed", or "failed" when TLS was tried. // // Redirects are not followed — a cross-scheme redirect would silently hop // between the gateway's cleartext and TLS legs, flipping the very behavior @@ -455,7 +460,24 @@ func fetch(w http.ResponseWriter, r *http.Request) { for name, values := range headers { req.Header[name] = values } + var handshake atomic.Value + // An HTTP failure alone can pass a denial test because of an unrelated + // certificate error or a close after TLS. Report handshake closure separately. + req = req.WithContext(httptrace.WithClientTrace(req.Context(), &httptrace.ClientTrace{ + TLSHandshakeDone: func(_ tls.ConnectionState, err error) { + result := "complete" + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) { + result = "closed" + } else if err != nil { + result = "failed" + } + handshake.Store(result) + }, + })) res, err := client.Do(req) + if result, ok := handshake.Load().(string); ok { + resp["tls_handshake"] = result + } if err != nil { resp["error"] = err.Error() writeJSON(w, resp) diff --git a/internal/e2e/fixtures/probe/main_test.go b/internal/e2e/fixtures/probe/main_test.go index 99ac3bb6a7..7d26c6a151 100644 --- a/internal/e2e/fixtures/probe/main_test.go +++ b/internal/e2e/fixtures/probe/main_test.go @@ -15,8 +15,10 @@ package main import ( + "crypto/tls" "encoding/json" "encoding/pem" + "errors" "net" "net/http" "net/http/httptest" @@ -296,8 +298,14 @@ func TestFetchTrustsOnlyTheGivenCA(t *testing.T) { if resp["error"] != "" || resp["status"] != "200" { t.Errorf("fetch = status %q, error %q; want 200", resp["status"], resp["error"]) } + if resp["tls_handshake"] != "complete" { + t.Errorf("TLS handshake = %q, want complete", resp["tls_handshake"]) + } return } + if resp["tls_handshake"] != "failed" { + t.Errorf("TLS handshake = %q, want failed", resp["tls_handshake"]) + } if !strings.Contains(resp["error"], "certificate") { t.Errorf("error = %q, want a certificate verification failure", resp["error"]) } @@ -305,6 +313,27 @@ func TestFetchTrustsOnlyTheGivenCA(t *testing.T) { } } +func TestFetchReportsClientHelloClosure(t *testing.T) { + origin := httptest.NewUnstartedServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Error("denied TLS reached HTTP") + })) + origin.TLS = &tls.Config{ + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + if err := hello.Conn.Close(); err != nil { + t.Errorf("closing denied TLS connection: %v", err) + } + return nil, errors.New("TLS destination denied") + }, + } + origin.StartTLS() + defer origin.Close() + + resp := doFetch(t, origin.URL) + if resp["tls_handshake"] != "closed" || resp["status"] != "" || resp["error"] == "" { + t.Fatalf("fetch = %v, want connection closed during TLS with no HTTP response", resp) + } +} + // roots=pem and ca come together, and ca must hold a certificate; anything // else fails before the request is sent. func TestFetchRejectsMismatchedCA(t *testing.T) { diff --git a/internal/e2e/suites/egressmitm/egressmitm_test.go b/internal/e2e/suites/egressmitm/egressmitm_test.go index 0df254c8e2..2d6da343b3 100644 --- a/internal/e2e/suites/egressmitm/egressmitm_test.go +++ b/internal/e2e/suites/egressmitm/egressmitm_test.go @@ -158,29 +158,18 @@ func TestActorEgressMITMTrust(t *testing.T) { t.Errorf("passthrough fetch %s dialed at %s: status %s, want 200", passthroughOrigin, unreachableAddress, bySNI.Status) } - // Envoy closes denied connections at the ClientHello; agentgateway - // terminates TLS to return 403. Trust the gateway's CA so a certificate - // failure cannot masquerade as either policy-denial response. - // - // Each name is an origin's fully qualified alias: rules match names - // exactly, so only the wrong-port rule lists the first and none the - // second. A gateway that relayed either would hand the actor a - // certificate the bundle rejects, and the dial keeps the actor off DNS. + // Each name is an origin's fully qualified alias: only the wrong-port rule + // lists the first and none lists the second. Neither may complete TLS. + // Certificate-verification failures must not count as policy refusals. + // The dial bypasses DNS. for _, tc := range []struct{ name, origin, dial string }{ {"passthrough rule excludes port 443", "https://" + passthrough.SvcName() + ".cluster.local/healthz", passthrough.TLSAddress()}, {"host outside the policy", "https://" + mitm.SvcName() + ".cluster.local/healthz", mitm.TLSAddress()}, } { t.Run(tc.name, func(t *testing.T) { denied := probeFetch(t, ctx, rc, id, tc.origin, "bundle", "dial="+url.QueryEscape(tc.dial)) - switch { - case denied.Status == "403" && denied.Error == "": - // An explicit denial after completing TLS with the actor. - case denied.Error == "": - t.Errorf("fetch of %s returned status %s, want 403 or a connection closed at the ClientHello", tc.origin, denied.Status) - case strings.Contains(denied.Error, "certificate") || strings.Contains(denied.Error, "x509"): - t.Errorf("fetch of %s failed certificate verification instead of returning a policy denial: %s", tc.origin, denied.Error) - case denied.Status != "": - t.Errorf("fetch of %s got status %s with error %q, want no HTTP exchange after a TLS failure", tc.origin, denied.Status, denied.Error) + if denied.TLSHandshake != "closed" || denied.Status != "" || denied.Error == "" { + t.Errorf("fetch of %s: handshake %q, status %q, error %q; want connection closed during TLS with no HTTP response", tc.origin, denied.TLSHandshake, denied.Status, denied.Error) } }) } @@ -213,9 +202,10 @@ const ( ) type fetchResponse struct { - Status string `json:"status"` - Error string `json:"error"` - Body string `json:"body"` + Status string `json:"status"` + Error string `json:"error"` + Body string `json:"body"` + TLSHandshake string `json:"tls_handshake"` } // reachWindow bounds the retries of a fetch that must return 200. diff --git a/internal/e2e/suites/ingressauth/ingressauth_test.go b/internal/e2e/suites/ingressauth/ingressauth_test.go index 155e76d28c..43b7b722d1 100644 --- a/internal/e2e/suites/ingressauth/ingressauth_test.go +++ b/internal/e2e/suites/ingressauth/ingressauth_test.go @@ -28,10 +28,13 @@ import ( "crypto/x509" "errors" "fmt" + "io" "math/big" "net" "net/http" + "net/http/httptest" "net/url" + "syscall" "testing" "time" @@ -49,8 +52,8 @@ import ( const unlistedServiceAccount = "router-client-unlisted" // targetActor names an actor that does not exist. A client the router lets -// through gets an HTTP response, whatever its status; one it turns away gets -// none, because the TLS handshake fails first. +// through reaches actor routing and gets an HTTP response. Authorization +// must reject an unlisted client before that lookup, even inside CONNECT. var targetActor = resources.ActorRef{Atespace: "ate-e2e-ingressauth", Name: "nobody"} // atenet-router's Service ports; see manifests/ate-install/atenet-router.yaml. @@ -65,14 +68,13 @@ type tlsListener struct { name string servicePort int32 // request sends one request for targetActor over a TLS connection - // presenting cert (nil for none), and returns the HTTP status. An error - // means no HTTP response came back. + // presenting cert (nil for none), including inside an accepted CONNECT + // tunnel, and returns the HTTP status. request func(ctx context.Context, address string, cfg *tls.Config) (int, error) } -// tlsListeners are both TLS ingress listeners. Every check runs against both: -// they share a transport socket, but nothing else would notice one of them -// losing it. +// tlsListeners are both TLS ingress listeners. Every check runs against both +// so one listener losing client authentication cannot hide behind the other. var tlsListeners = []tlsListener{ {name: "https", servicePort: httpsServicePort, request: httpsRequest}, {name: "connect-tls", servicePort: connectTLSServicePort, request: connectRequest}, @@ -135,14 +137,97 @@ func connectRequest(ctx context.Context, address string, cfg *tls.Config) (int, if err := req.Write(conn); err != nil { return 0, err } - resp, err := http.ReadResponse(bufio.NewReader(conn), req) + reader := bufio.NewReader(conn) + resp, err := http.ReadResponse(reader, req) if err != nil { return 0, err } + if resp.StatusCode == http.StatusOK { + req, err = http.NewRequestWithContext(ctx, http.MethodGet, "http://"+destination+"/", nil) + if err != nil { + return 0, err + } + req.Header.Set(atenet.TargetActorHeader, targetActor.String()) + if err := req.Write(conn); err != nil { + return 0, err + } + resp, err = http.ReadResponse(reader, req) + if err != nil { + return 0, err + } + } resp.Body.Close() return resp.StatusCode, nil } +func TestConnectRequest(t *testing.T) { + for _, tc := range []struct { + name string + connectStatus int + innerStatus int + wantStatus int + wantRefused bool + }{ + {name: "CONNECT denied", connectStatus: http.StatusForbidden, wantStatus: http.StatusForbidden, wantRefused: true}, + {name: "inner denied", connectStatus: http.StatusOK, innerStatus: http.StatusForbidden, wantStatus: http.StatusForbidden, wantRefused: true}, + {name: "inner closed", connectStatus: http.StatusOK, wantRefused: true}, + {name: "actor lookup reached", connectStatus: http.StatusOK, innerStatus: http.StatusNotFound, wantStatus: http.StatusNotFound}, + {name: "inner accepted", connectStatus: http.StatusOK, innerStatus: http.StatusOK, wantStatus: http.StatusOK}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodConnect { + t.Errorf("outer method = %s, want CONNECT", req.Method) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if tc.connectStatus != http.StatusOK { + w.WriteHeader(tc.connectStatus) + return + } + conn, rw, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Errorf("hijacking CONNECT: %v", err) + return + } + defer conn.Close() + if _, err := rw.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil { + t.Errorf("writing CONNECT response: %v", err) + return + } + if err := rw.Flush(); err != nil { + t.Errorf("flushing CONNECT response: %v", err) + return + } + inner, err := http.ReadRequest(rw.Reader) + if err != nil { + t.Errorf("reading inner request: %v", err) + return + } + defer inner.Body.Close() + if inner.Method != http.MethodGet || inner.Header.Get(atenet.TargetActorHeader) != targetActor.String() { + t.Errorf("unexpected inner request: %s, actor %q", inner.Method, inner.Header.Get(atenet.TargetActorHeader)) + } + if tc.innerStatus != 0 { + if _, err := fmt.Fprintf(conn, "HTTP/1.1 %d %s\r\nContent-Length: 0\r\n\r\n", tc.innerStatus, http.StatusText(tc.innerStatus)); err != nil { + t.Errorf("writing inner response: %v", err) + } + } + })) + defer server.Close() + roots := x509.NewCertPool() + roots.AddCert(server.Certificate()) + status, err := connectRequest(t.Context(), server.Listener.Addr().String(), &tls.Config{RootCAs: roots}) + if status != tc.wantStatus { + t.Errorf("status = %d, want %d (error: %v)", status, tc.wantStatus, err) + } + if got := ingressRefused(status, err, true); got != tc.wantRefused { + t.Errorf("refused = %t, want %t (error: %v)", got, tc.wantRefused, err) + } + }) + } +} + // routerTLSConfig is the TLS config a client presenting cert dials the router // with. It presents cert unconditionally: Go's client otherwise withholds a // certificate that does not chain to a CA the server's CertificateRequest @@ -173,19 +258,16 @@ func isRemoteTLSAlert(err error) bool { const refusalAttempts = 3 // TestIngressAuthentication checks each kind of client against both TLS -// listeners. +// listeners. Certificate trust is checked during TLS; the URI SAN allowlist +// may be enforced during TLS or by network authorization afterward. // // Each refused client gets a port-forward of its own, opened fresh and first // shown to carry the allowlisted client through: a port-forward tears down its // whole tunnel once one forwarded connection fails, so a shared one would turn -// every check after the first refusal into a dial error. The refusal itself -// has to be a TLS alert from the router, which a broken tunnel cannot fake; a -// failure without one is retried on a fresh port-forward, refusalAttempts -// times at most. +// every check after the first refusal into a dial error. Missing or untrusted +// certificates must receive a TLS alert. A trusted but unlisted identity may +// instead receive a connection close or HTTP 403 before actor routing. func TestIngressAuthentication(t *testing.T) { - if !e2e.CurrentAtenetDataplane().AuthenticatesIngressClients() { - t.Skip("the atenet dataplane does not authenticate ingress clients") - } ctx := context.Background() allowed := e2e.RouterClientCertificate(t, ctx, e2e.RouterClientServiceAccount) @@ -195,11 +277,12 @@ func TestIngressAuthentication(t *testing.T) { for _, listener := range tlsListeners { t.Run(listener.name, func(t *testing.T) { for _, tc := range []struct { - name string - cert *tls.Certificate + name string + cert *tls.Certificate + allowPostTLS bool }{ {name: "no client certificate", cert: nil}, - {name: "podidentity certificate for an unlisted SPIFFE ID", cert: &unlisted}, + {name: "podidentity certificate for an unlisted SPIFFE ID", cert: &unlisted, allowPostTLS: true}, {name: "allowlisted SPIFFE ID signed by an untrusted CA", cert: &foreign}, } { t.Run(tc.name, func(t *testing.T) { @@ -213,17 +296,24 @@ func TestIngressAuthentication(t *testing.T) { t.Logf("allowlisted client let through (status %d)", status) status, err = listener.request(ctx, address, routerTLSConfig(t, ctx, tc.cert)) - if err == nil { - t.Fatalf("router let the client through (status %d); want the TLS handshake refused", status) - } - if isRemoteTLSAlert(err) { + if ingressRefused(status, err, tc.allowPostTLS) { t.Logf("refused: %v", err) + if tc.allowPostTLS { + // A close alone could be a broken port-forward. + address = forwardRouterPort(t, ctx, listener.servicePort) + if _, err := listener.request(ctx, address, routerTLSConfig(t, ctx, &allowed)); err != nil { + t.Fatalf("allowlisted client failed after unlisted client refusal: %v", err) + } + } return } + if err == nil { + t.Fatalf("router let the client through (status %d); want client refusal", status) + } if attempt == refusalAttempts { - t.Fatalf("request failed without a TLS alert from the router %d times, so the refusal is unproven: %v", attempt, err) + t.Fatalf("request failed without proving client refusal %d times: %v", attempt, err) } - t.Logf("attempt %d failed without a TLS alert from the router; retrying on a fresh port-forward: %v", attempt, err) + t.Logf("attempt %d failed without proving client refusal; retrying on a fresh port-forward: %v", attempt, err) } }) } @@ -231,6 +321,49 @@ func TestIngressAuthentication(t *testing.T) { } } +func ingressRefused(status int, err error, allowPostTLS bool) bool { + if isRemoteTLSAlert(err) { + return true + } + if !allowPostTLS { + return false + } + if err == nil { + return status == http.StatusForbidden + } + return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) +} + +func TestIngressRefused(t *testing.T) { + for _, tc := range []struct { + name string + status int + err error + allowPostTLS bool + want bool + }{ + {name: "TLS alert", err: &net.OpError{Op: "remote error", Err: errors.New("bad certificate")}, want: true}, + {name: "unlisted identity closed", err: io.EOF, allowPostTLS: true, want: true}, + {name: "unlisted identity reset", err: syscall.ECONNRESET, allowPostTLS: true, want: true}, + {name: "unlisted identity broken pipe", err: syscall.EPIPE, allowPostTLS: true, want: true}, + {name: "unlisted identity forbidden", status: http.StatusForbidden, allowPostTLS: true, want: true}, + {name: "missing certificate closed", err: io.EOF}, + {name: "missing certificate broken pipe", err: syscall.EPIPE}, + {name: "untrusted certificate forbidden", status: http.StatusForbidden}, + {name: "dial failure", err: syscall.ECONNREFUSED, allowPostTLS: true}, + {name: "timeout", err: context.DeadlineExceeded, allowPostTLS: true}, + {name: "actor route reached", status: http.StatusNotFound, allowPostTLS: true}, + {name: "CONNECT accepted", status: http.StatusOK, allowPostTLS: true}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := ingressRefused(tc.status, tc.err, tc.allowPostTLS); got != tc.want { + t.Errorf("ingressRefused() = %t, want %t", got, tc.want) + } + }) + } +} + // foreignCertificate returns a client certificate carrying spiffeID, signed // by a throwaway CA the router does not trust. func foreignCertificate(t *testing.T, spiffeID string) tls.Certificate { diff --git a/internal/e2e/suites/networking/egresspolicy_test.go b/internal/e2e/suites/networking/egresspolicy_test.go index 31515a3bd5..ad2d07e704 100644 --- a/internal/e2e/suites/networking/egresspolicy_test.go +++ b/internal/e2e/suites/networking/egresspolicy_test.go @@ -168,7 +168,7 @@ func TestActorEgressHTTPSByHostname(t *testing.T) { if !dataplane.IsEgressPolicyDenied(status, string(body)) { t.Fatalf("fetch of %s, outside the policy, returned HTTP %d, want an egress-policy denial; body: %s", url, status, body) } - // Envoy denies by closing the connection, which the actor reports as + // Denied TLS closes at ClientHello, which the actor reports as // the same 502 as any failed fetch; a certificate error in it means the // handshake reached the origin. if got := decodeEgressFetch(t, body); strings.Contains(got.Error, "x509") || strings.Contains(got.Error, "certificate") { @@ -183,7 +183,7 @@ func TestActorEgressHTTPSByHostname(t *testing.T) { func TestActorEgressPassthroughPolicyRefresh(t *testing.T) { ctx := t.Context() dataplane := e2e.CurrentAtenetDataplane() - _, address, rootCA := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ + origin, address, rootCA := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ Name: "passthrough-origin", ImportPath: "go.ate.dev/substrate/internal/e2e/fixtures/testserver", Args: []string{"http"}, @@ -220,7 +220,8 @@ func TestActorEgressPassthroughPolicyRefresh(t *testing.T) { // Allow connection to be fully established before changing the policy. assertEgressGatewayConnect(t, ctx, since, actorAtespace, actorName, "443") - e2e.EnsureEgressPolicy(t, ctx, e2e.GetClients(), actorRef.ToObjectRef(), e2e.EgressAllowPassthrough("foo.bar.baz")) + // Keep DNS authorized so the second connection tests the PEP, not the DNS relay. + e2e.EnsureEgressPolicy(t, ctx, e2e.GetClients(), actorRef.ToObjectRef(), e2e.EgressAllowHTTP(origin.SvcName()+".cluster.local")) // Wait for the policy cached in ext_proc server to expire, so it will fetch the new one. // ext_proc cache freshness interval is 10 seconds. time.Sleep(11 * time.Second) diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index 6497fe35e4..d26489c09c 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -378,9 +378,9 @@ func assertEgressGatewayConnect(t *testing.T, ctx context.Context, since metav1. } return true case "agentgateway": - if strings.Contains(line.text, "CONNECT tunnel terminated") && - strings.Contains(line.text, "target=") && - strings.Contains(line.text, ":"+port) { + target, ok := accessLogField(line.text, "target") + if ok && strings.HasSuffix(target, ":"+port) && + protocolGatewayAccessLogMatches(line, resources.ActorRef{Atespace: atespace, Name: actorName}, target, true) { t.Logf("egress gateway tunneled the request: %s", line.text) return true } diff --git a/manifests/ate-install/components/agentgateway-egress/kustomization.yaml b/manifests/ate-install/components/agentgateway-egress/kustomization.yaml index 28e0012914..03a06a9177 100644 --- a/manifests/ate-install/components/agentgateway-egress/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway-egress/kustomization.yaml @@ -63,7 +63,7 @@ patches: mode: tunnel accessLog: add: - substrate.connect.authority: source.connectHeaders["host"] + substrate.connect.authority: 'destination.address + ":" + string(destination.port)' substrateEgressActorResolution: host: api.ate-system.svc:443 policies: @@ -139,7 +139,7 @@ patches: body: WebSocket egress is not supported - backends: - dynamic: - target: source.connectHeaders["host"] + target: 'destination.address + ":" + string(destination.port)' policies: substrateEgress: *egressPolicy - protocol: TLS diff --git a/manifests/ate-install/components/agentgateway-static-mtls/configmap.yaml b/manifests/ate-install/components/agentgateway-static-mtls/configmap.yaml new file mode 100644 index 0000000000..904028ea21 --- /dev/null +++ b/manifests/ate-install/components/agentgateway-static-mtls/configmap.yaml @@ -0,0 +1,120 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: atenet-router-agentgateway-config + namespace: ate-system +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + config: + statsAddr: 0.0.0.0:15020 + backend: + # See the ingress-only loopback rationale in ../agentgateway/configmap.yaml. + allowDynamicLoopback: true + + frontendPolicies: + networkAuthorization: + rules: + - allow: >- + source.spiffeId in [ATE_INGRESS_ALLOWED_SPIFFE_IDS] + tracing: + host: $AGENTGATEWAY_OTLP_ADDRESS + protocol: grpc + randomSampling: 0.01 + + gateways: + https: + port: 8443 + protocol: HTTPS + tls: &ingress-tls + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + root: /run/podidentity.podcert.ate.dev/trust-bundle.pem + + backends: + - name: dynamic + dynamic: {} + policies: + http: + version: HTTP/1.1 + backendTunnel: + proxy: + backend: /dynamic + mode: connect + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/podidentity.podcert.ate.dev/trust-bundle.pem + insecureHost: true + + routes: + - name: substrate-actors + gateways: + - https + matches: + - path: + pathPrefix: / + policies: &substrate-ingress + substrateIngress: + host: api.ate-system.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + - name: substrate-actors-grpc + gateways: + - https + matches: &grpc-matches + - method: POST + headers: + - name: content-type + value: + regex: '(?i)^application/grpc([+;].*)?$' + policies: *substrate-ingress + backends: &grpc-backends + - backend: /dynamic + policies: + http: + version: HTTP/2.0 + + binds: + - port: 8444 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: *ingress-tls + routes: [] + - mode: internal + listeners: + - protocol: HTTP + routes: + - name: substrate-actors-tunneled + matches: + - path: + pathPrefix: / + policies: *substrate-ingress + backends: + - backend: /dynamic + - name: substrate-actors-tunneled-grpc + matches: *grpc-matches + policies: *substrate-ingress + backends: *grpc-backends diff --git a/manifests/ate-install/components/agentgateway-static-mtls/kustomization.yaml b/manifests/ate-install/components/agentgateway-static-mtls/kustomization.yaml new file mode 100644 index 0000000000..58b39c1aad --- /dev/null +++ b/manifests/ate-install/components/agentgateway-static-mtls/kustomization.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +patches: + - path: configmap.yaml + target: + version: v1 + kind: ConfigMap + name: atenet-router-agentgateway-config + namespace: ate-system diff --git a/manifests/ate-install/components/agentgateway/configmap.yaml b/manifests/ate-install/components/agentgateway/configmap.yaml index 38bbe9b44f..2216e88520 100644 --- a/manifests/ate-install/components/agentgateway/configmap.yaml +++ b/manifests/ate-install/components/agentgateway/configmap.yaml @@ -22,6 +22,14 @@ data: # yaml-language-server: $schema=https://agentgateway.dev/schema/config config: statsAddr: 0.0.0.0:15020 + backend: + # The dynamic-backend guard checks CONNECT authority before tunneling. + # substrateIngress resolves ate-target-actor through ate-api-server to + # select the worker's mTLS atunnel endpoint. Atunnel selects that actor's + # dialer by ate-target-actor and uses only the authority's port, not its + # host, so a loopback authority cannot dial the router's own loopback. + # Keep this opt-in ingress-only, with all backends tunneled to atunnel. + allowDynamicLoopback: true frontendPolicies: tracing: @@ -142,7 +150,7 @@ data: mode: tunnel accessLog: add: - substrate.connect.authority: source.connectHeaders["host"] + substrate.connect.authority: 'destination.address + ":" + string(destination.port)' substrateEgressActorResolution: host: api.ate-system.svc:443 policies: @@ -172,7 +180,7 @@ data: tcpRoutes: - backends: - dynamic: - target: source.connectHeaders["host"] + target: 'destination.address + ":" + string(destination.port)' - protocol: HTTP routes: - name: reject-websocket-upgrade @@ -190,7 +198,7 @@ data: body: WebSocket egress is not supported - backends: - dynamic: - target: source.connectHeaders["host"] + target: 'destination.address + ":" + string(destination.port)' policies: substrateEgress: host: api.ate-system.svc:443 diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index 6765bb9491..fb6b70548d 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -42,7 +42,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: cr.agentgateway.dev/agentgateway:v0.0.0-alpha.9d36620d@sha256:bcfa0e5bfa8a4e8206ec4d2029f6b33a3599e55a9be81e1584ed4586298db3da + image: cr.agentgateway.dev/agentgateway:v1.7.0@sha256:60216aafb03d37f94e952ff3d51d2f630a65b10ecaebdc0a864882f92610c45b args: - -f - /etc/agentgateway/config.yaml @@ -118,7 +118,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: cr.agentgateway.dev/agentgateway:v0.0.0-alpha.9d36620d@sha256:bcfa0e5bfa8a4e8206ec4d2029f6b33a3599e55a9be81e1584ed4586298db3da + image: cr.agentgateway.dev/agentgateway:v1.7.0@sha256:60216aafb03d37f94e952ff3d51d2f630a65b10ecaebdc0a864882f92610c45b args: - -f - /etc/agentgateway/config.yaml From b5ae011c1ca1dfd3d3fe654cbcffc2e0acdb9b76 Mon Sep 17 00:00:00 2001 From: Shruti Nair Date: Tue, 27 Oct 2026 18:28:43 +0000 Subject: [PATCH 16/53] Document ReportActorCrashed in the WorkerService authz table (#3324) `ReportActorCrashed` was missing from the `WorkerService` table in `docs/authorization.md`. Fixes #3234 I used an AI agent to find the missing row and check it against registry.go and workerservice/crash.go. Signed-off-by: Shruti Nair --- docs/authorization.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/authorization.md b/docs/authorization.md index 5de12f020a..ab5cb19cd5 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -220,6 +220,7 @@ certificate. | `AcquireWorkerEpoch` | `atelet` on the worker's node | | `RegisterWorker` | `atelet` on the worker's node | | `SetWorkerSandboxRuntimes` | `atelet` on the worker's node | +| `ReportActorCrashed` | `atelet` on the node of the worker hosting the actor | | `RequestActorSuspend` | `atelet` on the node of the worker hosting the actor | | `MintWorkerActorJwt` | `atelet` on the node hosting the actor | | `MintWorkerActorCertificate` | `atelet` on the node of the worker the actor is assigned to | From 8f664b59f448c4e911d732a38e7e16fe2e0137fa Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 27 Oct 2026 21:24:07 +0000 Subject: [PATCH 17/53] MicroVM guests get their whole memory limit, and placement books each actor's runtime overhead (#3104) Today the microVM worker takes a flat 128MiB out of each actor's memory limit for the VMM, which is too much for small guests and too little for large ones, and nothing accounts for what an actor costs outside its sandbox when placing it. - Workers report what an actor costs on top of its limits on each runtime (`WorkerSandboxRuntime.actor_overhead`: a fixed amount plus an amount per GiB of the actor's memory limit), and placement books it with the actor, so it shows in the worker's allocated resources - MicroVM guests get their whole memory limit, and the worker reports 128MiB + 8MiB/GiB per actor as overhead (cloud-hypervisor, virtiofsd, host page tables, and the page cache of snapshot writes). Once a follow-up caps snapshot writes this drops to 4MiB + 8MiB/GiB - Both workers keep 64MiB of their memory limit out of the capacity they report, for the worker process and in-flight snapshot writes #3091 proposed wrapping `SandboxRuntime`, but #3022 already split the worker-side runtime (`WorkerSandboxRuntime`) from the stamp on snapshots and assignments (`ActorSandboxRuntime`), so the overhead is a field on the former. Same intent, and it stays out of compatibility matching. Fixes #3091 TODO: For microVM, actors still lose some memory inside the guest limit to various things, including the kernel and kata-agent. Similarly, in gVisor they lose memory to the sentry, which is inside the actor's cgroup. As such, we currently leave gVisor overhead accounting at zero. AI use: AI-assisted, I reviewed and tested it. Tested with a fresh install on GKE (gVisor and microVM e2e lanes). Breaking change: worker pools hold a little less. A worker's memory capacity is now its memory limit minus 64MiB, and each microVM actor books its limit plus 128MiB + 8MiB/GiB. A pool sized to exactly fit its actors (e.g. pod memory limit equal to one actor's limit) needs its `limits.memory` raised, roughly 64MiB per worker plus the microVM overhead per actor. MicroVM guests also now see their whole memory limit, 128MiB more than before. **These values are subject to tuning in the future**. - [x] Tests pass - [x] Docs update --------- Signed-off-by: Benjamin Elder --- api/ateapi/v1/ateapi.pb.go | 727 ++++++++++-------- api/ateapi/v1/ateapi.proto | 35 +- benchmarking/automation/orchestrator.py | 4 +- benchmarking/automation/tests.yaml | 14 +- benchmarking/deploy_locust.sh | 4 +- benchmarking/workloads/deploy.sh | 14 +- .../manifests/sleep-template.yaml.tmpl | 3 +- .../apivalidation/zz_generated.validation.go | 107 +++ .../internal/controlapi/workflow_resume.go | 32 +- .../controlapi/workflow_resume_test.go | 24 + cmd/ateapi/internal/scheduling/scheduling.go | 66 +- .../internal/scheduling/scheduling_test.go | 86 +++ .../apivalidation/zz_generated.validation.go | 107 +++ cmd/atelet/workersupport.go | 14 +- cmd/atelet/workersupport_test.go | 15 + cmd/ateworker-gvisor/main.go | 8 + cmd/ateworker-microvm/main.go | 47 +- cmd/ateworker-microvm/run.go | 68 +- cmd/ateworker-microvm/run_test.go | 19 +- cmd/ateworker-microvm/spec.go | 31 +- cmd/ateworker-microvm/spec_test.go | 44 +- .../counter-microvm-template.yaml.tmpl | 5 +- demos/counter/counter-microvm.yaml.tmpl | 7 +- docs/api-guide.md | 7 +- internal/ateworker/register/register.go | 47 +- internal/ateworker/register/register_test.go | 76 +- .../scripts/personal-assistant-light.yaml | 2 +- internal/e2e/suites/sizing/sizing_test.go | 7 +- internal/hardware/hardware.go | 35 +- internal/hardware/hardware_test.go | 43 ++ internal/proto/ateletpb/atelet.pb.go | 725 +++++++++-------- internal/proto/ateletpb/atelet.proto | 25 + 32 files changed, 1611 insertions(+), 837 deletions(-) diff --git a/api/ateapi/v1/ateapi.pb.go b/api/ateapi/v1/ateapi.pb.go index 8b0cf6da99..b61e08acf3 100644 --- a/api/ateapi/v1/ateapi.pb.go +++ b/api/ateapi/v1/ateapi.pb.go @@ -7492,7 +7492,8 @@ type WorkerStatus struct { // +k8s:optional Capacity *WorkerResources `protobuf:"bytes,2,opt,name=capacity,proto3" json:"capacity,omitempty"` // The allocated resources of the Worker. This is the sum of all the - // resources consumed by the Actors currently assigned to this Worker. + // resources consumed by the Actors currently assigned to this Worker, + // including their runtime's actor_overhead. // // +k8s:optional Allocated *WorkerResources `protobuf:"bytes,3,opt,name=allocated,proto3" json:"allocated,omitempty"` @@ -7722,7 +7723,15 @@ type WorkerSandboxRuntime struct { // restore snapshots from. // // +k8s:required - Version *VersionedSandboxCompat `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + Version *VersionedSandboxCompat `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + // The resource overhead incurred by a Worker for each Actor running under + // this runtime, on top of the Actor's stated limits. Placement considers + // this when deciding if a Worker has enough capacity for an Actor, and it is + // accounted for in allocated. If not specified, Actors under this runtime + // are assumed to have negligible overhead. + // + // +k8s:optional + ActorOverhead *RuntimeOverhead `protobuf:"bytes,2,opt,name=actor_overhead,json=actorOverhead,proto3" json:"actor_overhead,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -7764,6 +7773,79 @@ func (x *WorkerSandboxRuntime) GetVersion() *VersionedSandboxCompat { return nil } +func (x *WorkerSandboxRuntime) GetActorOverhead() *RuntimeOverhead { + if x != nil { + return x.ActorOverhead + } + return nil +} + +// RuntimeOverhead is the memory an Actor uses on its Worker outside its own +// limits: a fixed amount, plus an amount in proportion to its memory limit. +type RuntimeOverhead struct { + state protoimpl.MessageState `protogen:"open.v1"` + // memory_bytes is a fixed amount of memory for each Actor. + // + // +k8s:optional + // +k8s:minimum=0 + // +k8s:maximum=1099511627776 # 1TiB + MemoryBytes int64 `protobuf:"varint,1,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` + // memory_bytes_per_gib is memory per GiB of the Actor's memory limit, e.g. + // host page tables for a microVM's guest RAM. It scales in proportion and + // rounds down to a byte, so an Actor with a 512MiB limit adds half of it. + // + // +k8s:optional + // +k8s:minimum=0 + // +k8s:maximum=1073741824 # 1GiB + MemoryBytesPerGib int64 `protobuf:"varint,2,opt,name=memory_bytes_per_gib,json=memoryBytesPerGib,proto3" json:"memory_bytes_per_gib,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RuntimeOverhead) Reset() { + *x = RuntimeOverhead{} + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[107] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RuntimeOverhead) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RuntimeOverhead) ProtoMessage() {} + +func (x *RuntimeOverhead) ProtoReflect() protoreflect.Message { + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[107] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RuntimeOverhead.ProtoReflect.Descriptor instead. +func (*RuntimeOverhead) Descriptor() ([]byte, []int) { + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{107} +} + +func (x *RuntimeOverhead) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + +func (x *RuntimeOverhead) GetMemoryBytesPerGib() int64 { + if x != nil { + return x.MemoryBytesPerGib + } + return 0 +} + // ActorSandboxRuntime is the runtime an actor runs under, as recorded on its // assignment and on the snapshots taken from it: one class and one of the // compatibility versions a Worker reports for it. @@ -7786,7 +7868,7 @@ type ActorSandboxRuntime struct { func (x *ActorSandboxRuntime) Reset() { *x = ActorSandboxRuntime{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[107] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[108] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7798,7 +7880,7 @@ func (x *ActorSandboxRuntime) String() string { func (*ActorSandboxRuntime) ProtoMessage() {} func (x *ActorSandboxRuntime) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[107] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[108] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7811,7 +7893,7 @@ func (x *ActorSandboxRuntime) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorSandboxRuntime.ProtoReflect.Descriptor instead. func (*ActorSandboxRuntime) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{107} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{108} } func (x *ActorSandboxRuntime) GetSandboxClass() string { @@ -7862,7 +7944,7 @@ type VersionedSandboxCompat struct { func (x *VersionedSandboxCompat) Reset() { *x = VersionedSandboxCompat{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[108] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[109] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7874,7 +7956,7 @@ func (x *VersionedSandboxCompat) String() string { func (*VersionedSandboxCompat) ProtoMessage() {} func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[108] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[109] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7887,7 +7969,7 @@ func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { // Deprecated: Use VersionedSandboxCompat.ProtoReflect.Descriptor instead. func (*VersionedSandboxCompat) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{108} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{109} } func (x *VersionedSandboxCompat) GetSchemaVersion() string { @@ -7923,7 +8005,7 @@ type AttributeEntry struct { func (x *AttributeEntry) Reset() { *x = AttributeEntry{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[109] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[110] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7935,7 +8017,7 @@ func (x *AttributeEntry) String() string { func (*AttributeEntry) ProtoMessage() {} func (x *AttributeEntry) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[109] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[110] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7948,7 +8030,7 @@ func (x *AttributeEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use AttributeEntry.ProtoReflect.Descriptor instead. func (*AttributeEntry) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{109} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{110} } func (x *AttributeEntry) GetKey() string { @@ -7995,7 +8077,7 @@ type WorkerResources struct { func (x *WorkerResources) Reset() { *x = WorkerResources{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[110] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[111] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8007,7 +8089,7 @@ func (x *WorkerResources) String() string { func (*WorkerResources) ProtoMessage() {} func (x *WorkerResources) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[110] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[111] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8020,7 +8102,7 @@ func (x *WorkerResources) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. func (*WorkerResources) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{110} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{111} } func (x *WorkerResources) GetCpuMillicores() int64 { @@ -8065,7 +8147,8 @@ type ActorAssignment struct { // +k8s:required // +k8s:format=k8s-uuid ActorUid string `protobuf:"bytes,3,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` - // The resources allocated to the Actor when it was assigned. + // The resources allocated to the Actor when it was assigned: its limits, + // plus the actor_overhead of the runtime it runs under on the Worker. // // +k8s:optional Resources *Resources `protobuf:"bytes,4,opt,name=resources,proto3" json:"resources,omitempty"` @@ -8075,7 +8158,7 @@ type ActorAssignment struct { func (x *ActorAssignment) Reset() { *x = ActorAssignment{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[111] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[112] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8087,7 +8170,7 @@ func (x *ActorAssignment) String() string { func (*ActorAssignment) ProtoMessage() {} func (x *ActorAssignment) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[111] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[112] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8100,7 +8183,7 @@ func (x *ActorAssignment) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorAssignment.ProtoReflect.Descriptor instead. func (*ActorAssignment) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{111} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{112} } func (x *ActorAssignment) GetMetadata() *ResourceMetadata { @@ -8145,7 +8228,7 @@ type AcquireWorkerEpochRequest struct { func (x *AcquireWorkerEpochRequest) Reset() { *x = AcquireWorkerEpochRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[112] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[113] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8157,7 +8240,7 @@ func (x *AcquireWorkerEpochRequest) String() string { func (*AcquireWorkerEpochRequest) ProtoMessage() {} func (x *AcquireWorkerEpochRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[112] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[113] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8170,7 +8253,7 @@ func (x *AcquireWorkerEpochRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AcquireWorkerEpochRequest.ProtoReflect.Descriptor instead. func (*AcquireWorkerEpochRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{112} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{113} } func (x *AcquireWorkerEpochRequest) GetWorker() *ObjectRef { @@ -8190,7 +8273,7 @@ type AcquireWorkerEpochResponse struct { func (x *AcquireWorkerEpochResponse) Reset() { *x = AcquireWorkerEpochResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[113] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[114] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8202,7 +8285,7 @@ func (x *AcquireWorkerEpochResponse) String() string { func (*AcquireWorkerEpochResponse) ProtoMessage() {} func (x *AcquireWorkerEpochResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[113] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[114] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8215,7 +8298,7 @@ func (x *AcquireWorkerEpochResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AcquireWorkerEpochResponse.ProtoReflect.Descriptor instead. func (*AcquireWorkerEpochResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{113} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{114} } func (x *AcquireWorkerEpochResponse) GetEpoch() int64 { @@ -8258,7 +8341,7 @@ type RegisterWorkerRequest struct { func (x *RegisterWorkerRequest) Reset() { *x = RegisterWorkerRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[114] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[115] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8270,7 +8353,7 @@ func (x *RegisterWorkerRequest) String() string { func (*RegisterWorkerRequest) ProtoMessage() {} func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[114] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[115] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8283,7 +8366,7 @@ func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerRequest.ProtoReflect.Descriptor instead. func (*RegisterWorkerRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{114} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{115} } func (x *RegisterWorkerRequest) GetWorker() *ObjectRef { @@ -8324,7 +8407,7 @@ type RegisterWorkerResponse struct { func (x *RegisterWorkerResponse) Reset() { *x = RegisterWorkerResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[115] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[116] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8336,7 +8419,7 @@ func (x *RegisterWorkerResponse) String() string { func (*RegisterWorkerResponse) ProtoMessage() {} func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[115] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[116] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8349,7 +8432,7 @@ func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerResponse.ProtoReflect.Descriptor instead. func (*RegisterWorkerResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{115} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{116} } func (x *RegisterWorkerResponse) GetWorker() *Worker { @@ -8389,7 +8472,7 @@ type SetWorkerSandboxRuntimesRequest struct { func (x *SetWorkerSandboxRuntimesRequest) Reset() { *x = SetWorkerSandboxRuntimesRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[116] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[117] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8401,7 +8484,7 @@ func (x *SetWorkerSandboxRuntimesRequest) String() string { func (*SetWorkerSandboxRuntimesRequest) ProtoMessage() {} func (x *SetWorkerSandboxRuntimesRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[116] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[117] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8414,7 +8497,7 @@ func (x *SetWorkerSandboxRuntimesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SetWorkerSandboxRuntimesRequest.ProtoReflect.Descriptor instead. func (*SetWorkerSandboxRuntimesRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{116} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{117} } func (x *SetWorkerSandboxRuntimesRequest) GetWorker() *ObjectRef { @@ -8448,7 +8531,7 @@ type SetWorkerSandboxRuntimesResponse struct { func (x *SetWorkerSandboxRuntimesResponse) Reset() { *x = SetWorkerSandboxRuntimesResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[117] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[118] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8460,7 +8543,7 @@ func (x *SetWorkerSandboxRuntimesResponse) String() string { func (*SetWorkerSandboxRuntimesResponse) ProtoMessage() {} func (x *SetWorkerSandboxRuntimesResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[117] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[118] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8473,7 +8556,7 @@ func (x *SetWorkerSandboxRuntimesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SetWorkerSandboxRuntimesResponse.ProtoReflect.Descriptor instead. func (*SetWorkerSandboxRuntimesResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{117} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{118} } func (x *SetWorkerSandboxRuntimesResponse) GetWorker() *Worker { @@ -8520,7 +8603,7 @@ type MintWorkerActorCertificateRequest struct { func (x *MintWorkerActorCertificateRequest) Reset() { *x = MintWorkerActorCertificateRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[118] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[119] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8532,7 +8615,7 @@ func (x *MintWorkerActorCertificateRequest) String() string { func (*MintWorkerActorCertificateRequest) ProtoMessage() {} func (x *MintWorkerActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[118] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[119] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8545,7 +8628,7 @@ func (x *MintWorkerActorCertificateRequest) ProtoReflect() protoreflect.Message // Deprecated: Use MintWorkerActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintWorkerActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{118} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{119} } func (x *MintWorkerActorCertificateRequest) GetActor() *ObjectRef { @@ -8589,7 +8672,7 @@ type MintWorkerActorCertificateResponse struct { func (x *MintWorkerActorCertificateResponse) Reset() { *x = MintWorkerActorCertificateResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[119] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[120] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8601,7 +8684,7 @@ func (x *MintWorkerActorCertificateResponse) String() string { func (*MintWorkerActorCertificateResponse) ProtoMessage() {} func (x *MintWorkerActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[119] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[120] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8614,7 +8697,7 @@ func (x *MintWorkerActorCertificateResponse) ProtoReflect() protoreflect.Message // Deprecated: Use MintWorkerActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintWorkerActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{119} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{120} } func (x *MintWorkerActorCertificateResponse) GetActorCertificates() [][]byte { @@ -8655,7 +8738,7 @@ type MintWorkerActorJwtRequest struct { func (x *MintWorkerActorJwtRequest) Reset() { *x = MintWorkerActorJwtRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[120] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[121] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8667,7 +8750,7 @@ func (x *MintWorkerActorJwtRequest) String() string { func (*MintWorkerActorJwtRequest) ProtoMessage() {} func (x *MintWorkerActorJwtRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[120] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[121] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8680,7 +8763,7 @@ func (x *MintWorkerActorJwtRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintWorkerActorJwtRequest.ProtoReflect.Descriptor instead. func (*MintWorkerActorJwtRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{120} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{121} } func (x *MintWorkerActorJwtRequest) GetActor() *ObjectRef { @@ -8728,7 +8811,7 @@ type MintWorkerActorJwtResponse struct { func (x *MintWorkerActorJwtResponse) Reset() { *x = MintWorkerActorJwtResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[121] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[122] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8740,7 +8823,7 @@ func (x *MintWorkerActorJwtResponse) String() string { func (*MintWorkerActorJwtResponse) ProtoMessage() {} func (x *MintWorkerActorJwtResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[121] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[122] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8753,7 +8836,7 @@ func (x *MintWorkerActorJwtResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintWorkerActorJwtResponse.ProtoReflect.Descriptor instead. func (*MintWorkerActorJwtResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{121} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{122} } func (x *MintWorkerActorJwtResponse) GetActorJwt() string { @@ -8801,7 +8884,7 @@ type RequestActorSuspendRequest struct { func (x *RequestActorSuspendRequest) Reset() { *x = RequestActorSuspendRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[122] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[123] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8813,7 +8896,7 @@ func (x *RequestActorSuspendRequest) String() string { func (*RequestActorSuspendRequest) ProtoMessage() {} func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[122] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[123] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8826,7 +8909,7 @@ func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendRequest.ProtoReflect.Descriptor instead. func (*RequestActorSuspendRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{122} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{123} } func (x *RequestActorSuspendRequest) GetWorker() *ObjectRef { @@ -8860,7 +8943,7 @@ type RequestActorSuspendResponse struct { func (x *RequestActorSuspendResponse) Reset() { *x = RequestActorSuspendResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[123] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[124] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8872,7 +8955,7 @@ func (x *RequestActorSuspendResponse) String() string { func (*RequestActorSuspendResponse) ProtoMessage() {} func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[123] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[124] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8885,7 +8968,7 @@ func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendResponse.ProtoReflect.Descriptor instead. func (*RequestActorSuspendResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{123} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{124} } func (x *RequestActorSuspendResponse) GetActor() *Actor { @@ -8925,7 +9008,7 @@ type ReportActorCrashedRequest struct { func (x *ReportActorCrashedRequest) Reset() { *x = ReportActorCrashedRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[124] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[125] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8937,7 +9020,7 @@ func (x *ReportActorCrashedRequest) String() string { func (*ReportActorCrashedRequest) ProtoMessage() {} func (x *ReportActorCrashedRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[124] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[125] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8950,7 +9033,7 @@ func (x *ReportActorCrashedRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorCrashedRequest.ProtoReflect.Descriptor instead. func (*ReportActorCrashedRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{124} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{125} } func (x *ReportActorCrashedRequest) GetWorker() *ObjectRef { @@ -8991,7 +9074,7 @@ type ReportActorCrashedResponse struct { func (x *ReportActorCrashedResponse) Reset() { *x = ReportActorCrashedResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[125] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[126] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9003,7 +9086,7 @@ func (x *ReportActorCrashedResponse) String() string { func (*ReportActorCrashedResponse) ProtoMessage() {} func (x *ReportActorCrashedResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[125] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[126] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9016,7 +9099,7 @@ func (x *ReportActorCrashedResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorCrashedResponse.ProtoReflect.Descriptor instead. func (*ReportActorCrashedResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{125} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{126} } func (x *ReportActorCrashedResponse) GetActor() *Actor { @@ -9047,7 +9130,7 @@ type ReportActorSnapshotStateRequest struct { func (x *ReportActorSnapshotStateRequest) Reset() { *x = ReportActorSnapshotStateRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[126] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[127] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9059,7 +9142,7 @@ func (x *ReportActorSnapshotStateRequest) String() string { func (*ReportActorSnapshotStateRequest) ProtoMessage() {} func (x *ReportActorSnapshotStateRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[126] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[127] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9072,7 +9155,7 @@ func (x *ReportActorSnapshotStateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorSnapshotStateRequest.ProtoReflect.Descriptor instead. func (*ReportActorSnapshotStateRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{126} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{127} } func (x *ReportActorSnapshotStateRequest) GetActor() *ObjectRef { @@ -9104,7 +9187,7 @@ type ReportActorSnapshotStateResponse struct { func (x *ReportActorSnapshotStateResponse) Reset() { *x = ReportActorSnapshotStateResponse{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[127] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[128] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9116,7 +9199,7 @@ func (x *ReportActorSnapshotStateResponse) String() string { func (*ReportActorSnapshotStateResponse) ProtoMessage() {} func (x *ReportActorSnapshotStateResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[127] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[128] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9129,7 +9212,7 @@ func (x *ReportActorSnapshotStateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorSnapshotStateResponse.ProtoReflect.Descriptor instead. func (*ReportActorSnapshotStateResponse) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{127} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{128} } // ActorSnapshotState is what atelet holds for one Actor on its node. It @@ -9163,7 +9246,7 @@ type ActorSnapshotState struct { func (x *ActorSnapshotState) Reset() { *x = ActorSnapshotState{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[128] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[129] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9175,7 +9258,7 @@ func (x *ActorSnapshotState) String() string { func (*ActorSnapshotState) ProtoMessage() {} func (x *ActorSnapshotState) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[128] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[129] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9188,7 +9271,7 @@ func (x *ActorSnapshotState) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorSnapshotState.ProtoReflect.Descriptor instead. func (*ActorSnapshotState) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{128} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{129} } func (x *ActorSnapshotState) GetLatestLocal() *SnapshotReport { @@ -9260,7 +9343,7 @@ type SnapshotReport struct { func (x *SnapshotReport) Reset() { *x = SnapshotReport{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[129] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[130] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9272,7 +9355,7 @@ func (x *SnapshotReport) String() string { func (*SnapshotReport) ProtoMessage() {} func (x *SnapshotReport) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[129] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[130] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9285,7 +9368,7 @@ func (x *SnapshotReport) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotReport.ProtoReflect.Descriptor instead. func (*SnapshotReport) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{129} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{130} } func (x *SnapshotReport) GetUuid() string { @@ -9355,7 +9438,7 @@ type AccessPolicy struct { func (x *AccessPolicy) Reset() { *x = AccessPolicy{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[130] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[131] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9367,7 +9450,7 @@ func (x *AccessPolicy) String() string { func (*AccessPolicy) ProtoMessage() {} func (x *AccessPolicy) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[130] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[131] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9380,7 +9463,7 @@ func (x *AccessPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use AccessPolicy.ProtoReflect.Descriptor instead. func (*AccessPolicy) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{130} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{131} } func (x *AccessPolicy) GetMetadata() *ResourceMetadata { @@ -9422,7 +9505,7 @@ type Binding struct { func (x *Binding) Reset() { *x = Binding{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[131] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[132] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9434,7 +9517,7 @@ func (x *Binding) String() string { func (*Binding) ProtoMessage() {} func (x *Binding) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[131] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[132] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9447,7 +9530,7 @@ func (x *Binding) ProtoReflect() protoreflect.Message { // Deprecated: Use Binding.ProtoReflect.Descriptor instead. func (*Binding) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{131} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{132} } func (x *Binding) GetRole() string { @@ -9479,7 +9562,7 @@ type GetGlobalAccessPolicyRequest struct { func (x *GetGlobalAccessPolicyRequest) Reset() { *x = GetGlobalAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[132] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[133] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9491,7 +9574,7 @@ func (x *GetGlobalAccessPolicyRequest) String() string { func (*GetGlobalAccessPolicyRequest) ProtoMessage() {} func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[132] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[133] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9504,7 +9587,7 @@ func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{132} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{133} } func (x *GetGlobalAccessPolicyRequest) GetName() string { @@ -9528,7 +9611,7 @@ type CreateGlobalAccessPolicyRequest struct { func (x *CreateGlobalAccessPolicyRequest) Reset() { *x = CreateGlobalAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[133] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[134] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9540,7 +9623,7 @@ func (x *CreateGlobalAccessPolicyRequest) String() string { func (*CreateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[133] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[134] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9553,7 +9636,7 @@ func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{133} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{134} } func (x *CreateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -9577,7 +9660,7 @@ type UpdateGlobalAccessPolicyRequest struct { func (x *UpdateGlobalAccessPolicyRequest) Reset() { *x = UpdateGlobalAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[134] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[135] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9589,7 +9672,7 @@ func (x *UpdateGlobalAccessPolicyRequest) String() string { func (*UpdateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[134] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[135] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9602,7 +9685,7 @@ func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{134} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{135} } func (x *UpdateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -9632,7 +9715,7 @@ type GetAtespaceAccessPolicyRequest struct { func (x *GetAtespaceAccessPolicyRequest) Reset() { *x = GetAtespaceAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[135] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[136] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9644,7 +9727,7 @@ func (x *GetAtespaceAccessPolicyRequest) String() string { func (*GetAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[135] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[136] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9657,7 +9740,7 @@ func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{135} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{136} } func (x *GetAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9693,7 +9776,7 @@ type CreateAtespaceAccessPolicyRequest struct { func (x *CreateAtespaceAccessPolicyRequest) Reset() { *x = CreateAtespaceAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[136] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[137] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9705,7 +9788,7 @@ func (x *CreateAtespaceAccessPolicyRequest) String() string { func (*CreateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[136] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[137] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9718,7 +9801,7 @@ func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use CreateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{136} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{137} } func (x *CreateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9754,7 +9837,7 @@ type UpdateAtespaceAccessPolicyRequest struct { func (x *UpdateAtespaceAccessPolicyRequest) Reset() { *x = UpdateAtespaceAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[137] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[138] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9766,7 +9849,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) String() string { func (*UpdateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[137] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[138] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9779,7 +9862,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use UpdateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{137} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{138} } func (x *UpdateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9820,7 +9903,7 @@ type DeleteAtespaceAccessPolicyRequest struct { func (x *DeleteAtespaceAccessPolicyRequest) Reset() { *x = DeleteAtespaceAccessPolicyRequest{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[138] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[139] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9832,7 +9915,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) String() string { func (*DeleteAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[138] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[139] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9845,7 +9928,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use DeleteAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*DeleteAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{138} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{139} } func (x *DeleteAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -9891,7 +9974,7 @@ type ResourceAnnotation struct { func (x *ResourceAnnotation) Reset() { *x = ResourceAnnotation{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[139] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[140] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9903,7 +9986,7 @@ func (x *ResourceAnnotation) String() string { func (*ResourceAnnotation) ProtoMessage() {} func (x *ResourceAnnotation) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[139] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[140] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9916,7 +9999,7 @@ func (x *ResourceAnnotation) ProtoReflect() protoreflect.Message { // Deprecated: Use ResourceAnnotation.ProtoReflect.Descriptor instead. func (*ResourceAnnotation) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{139} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{140} } func (x *ResourceAnnotation) GetParents() []string { @@ -9959,7 +10042,7 @@ type MethodAnnotation struct { func (x *MethodAnnotation) Reset() { *x = MethodAnnotation{} - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[140] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[141] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9971,7 +10054,7 @@ func (x *MethodAnnotation) String() string { func (*MethodAnnotation) ProtoMessage() {} func (x *MethodAnnotation) ProtoReflect() protoreflect.Message { - mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[140] + mi := &file_api_ateapi_v1_ateapi_proto_msgTypes[141] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9984,7 +10067,7 @@ func (x *MethodAnnotation) ProtoReflect() protoreflect.Message { // Deprecated: Use MethodAnnotation.ProtoReflect.Descriptor instead. func (*MethodAnnotation) Descriptor() ([]byte, []int) { - return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{140} + return file_api_ateapi_v1_ateapi_proto_rawDescGZIP(), []int{141} } func (x *MethodAnnotation) GetResource() string { @@ -10408,9 +10491,13 @@ const file_api_ateapi_v1_ateapi_proto_rawDesc = "" + "\x12WorkerSandboxClass\x12#\n" + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x12R\n" + "\x0fdefault_runtime\x18\x02 \x01(\v2).substrate.ateapi.v1.WorkerSandboxRuntimeR\x0edefaultRuntime\x12Z\n" + - "\x13restorable_runtimes\x18\x03 \x03(\v2).substrate.ateapi.v1.WorkerSandboxRuntimeR\x12restorableRuntimes\"]\n" + + "\x13restorable_runtimes\x18\x03 \x03(\v2).substrate.ateapi.v1.WorkerSandboxRuntimeR\x12restorableRuntimes\"\xaa\x01\n" + "\x14WorkerSandboxRuntime\x12E\n" + - "\aversion\x18\x01 \x01(\v2+.substrate.ateapi.v1.VersionedSandboxCompatR\aversion\"\x81\x01\n" + + "\aversion\x18\x01 \x01(\v2+.substrate.ateapi.v1.VersionedSandboxCompatR\aversion\x12K\n" + + "\x0eactor_overhead\x18\x02 \x01(\v2$.substrate.ateapi.v1.RuntimeOverheadR\ractorOverhead\"e\n" + + "\x0fRuntimeOverhead\x12!\n" + + "\fmemory_bytes\x18\x01 \x01(\x03R\vmemoryBytes\x12/\n" + + "\x14memory_bytes_per_gib\x18\x02 \x01(\x03R\x11memoryBytesPerGib\"\x81\x01\n" + "\x13ActorSandboxRuntime\x12#\n" + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x12E\n" + "\aversion\x18\x02 \x01(\v2+.substrate.ateapi.v1.VersionedSandboxCompatR\aversion\"\x84\x01\n" + @@ -10696,7 +10783,7 @@ func file_api_ateapi_v1_ateapi_proto_rawDescGZIP() []byte { } var file_api_ateapi_v1_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 10) -var file_api_ateapi_v1_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 144) +var file_api_ateapi_v1_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 145) var file_api_ateapi_v1_ateapi_proto_goTypes = []any{ (SnapshotOwner)(0), // 0: substrate.ateapi.v1.SnapshotOwner (SnapshotStorageState)(0), // 1: substrate.ateapi.v1.SnapshotStorageState @@ -10815,46 +10902,47 @@ var file_api_ateapi_v1_ateapi_proto_goTypes = []any{ (*WorkerStatus)(nil), // 114: substrate.ateapi.v1.WorkerStatus (*WorkerSandboxClass)(nil), // 115: substrate.ateapi.v1.WorkerSandboxClass (*WorkerSandboxRuntime)(nil), // 116: substrate.ateapi.v1.WorkerSandboxRuntime - (*ActorSandboxRuntime)(nil), // 117: substrate.ateapi.v1.ActorSandboxRuntime - (*VersionedSandboxCompat)(nil), // 118: substrate.ateapi.v1.VersionedSandboxCompat - (*AttributeEntry)(nil), // 119: substrate.ateapi.v1.AttributeEntry - (*WorkerResources)(nil), // 120: substrate.ateapi.v1.WorkerResources - (*ActorAssignment)(nil), // 121: substrate.ateapi.v1.ActorAssignment - (*AcquireWorkerEpochRequest)(nil), // 122: substrate.ateapi.v1.AcquireWorkerEpochRequest - (*AcquireWorkerEpochResponse)(nil), // 123: substrate.ateapi.v1.AcquireWorkerEpochResponse - (*RegisterWorkerRequest)(nil), // 124: substrate.ateapi.v1.RegisterWorkerRequest - (*RegisterWorkerResponse)(nil), // 125: substrate.ateapi.v1.RegisterWorkerResponse - (*SetWorkerSandboxRuntimesRequest)(nil), // 126: substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest - (*SetWorkerSandboxRuntimesResponse)(nil), // 127: substrate.ateapi.v1.SetWorkerSandboxRuntimesResponse - (*MintWorkerActorCertificateRequest)(nil), // 128: substrate.ateapi.v1.MintWorkerActorCertificateRequest - (*MintWorkerActorCertificateResponse)(nil), // 129: substrate.ateapi.v1.MintWorkerActorCertificateResponse - (*MintWorkerActorJwtRequest)(nil), // 130: substrate.ateapi.v1.MintWorkerActorJwtRequest - (*MintWorkerActorJwtResponse)(nil), // 131: substrate.ateapi.v1.MintWorkerActorJwtResponse - (*RequestActorSuspendRequest)(nil), // 132: substrate.ateapi.v1.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 133: substrate.ateapi.v1.RequestActorSuspendResponse - (*ReportActorCrashedRequest)(nil), // 134: substrate.ateapi.v1.ReportActorCrashedRequest - (*ReportActorCrashedResponse)(nil), // 135: substrate.ateapi.v1.ReportActorCrashedResponse - (*ReportActorSnapshotStateRequest)(nil), // 136: substrate.ateapi.v1.ReportActorSnapshotStateRequest - (*ReportActorSnapshotStateResponse)(nil), // 137: substrate.ateapi.v1.ReportActorSnapshotStateResponse - (*ActorSnapshotState)(nil), // 138: substrate.ateapi.v1.ActorSnapshotState - (*SnapshotReport)(nil), // 139: substrate.ateapi.v1.SnapshotReport - (*AccessPolicy)(nil), // 140: substrate.ateapi.v1.AccessPolicy - (*Binding)(nil), // 141: substrate.ateapi.v1.Binding - (*GetGlobalAccessPolicyRequest)(nil), // 142: substrate.ateapi.v1.GetGlobalAccessPolicyRequest - (*CreateGlobalAccessPolicyRequest)(nil), // 143: substrate.ateapi.v1.CreateGlobalAccessPolicyRequest - (*UpdateGlobalAccessPolicyRequest)(nil), // 144: substrate.ateapi.v1.UpdateGlobalAccessPolicyRequest - (*GetAtespaceAccessPolicyRequest)(nil), // 145: substrate.ateapi.v1.GetAtespaceAccessPolicyRequest - (*CreateAtespaceAccessPolicyRequest)(nil), // 146: substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest - (*UpdateAtespaceAccessPolicyRequest)(nil), // 147: substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest - (*DeleteAtespaceAccessPolicyRequest)(nil), // 148: substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest - (*ResourceAnnotation)(nil), // 149: substrate.ateapi.v1.ResourceAnnotation - (*MethodAnnotation)(nil), // 150: substrate.ateapi.v1.MethodAnnotation - nil, // 151: substrate.ateapi.v1.Selector.MatchLabelsEntry - nil, // 152: substrate.ateapi.v1.ExternalVolumeStatus.VolumeContextEntry - nil, // 153: substrate.ateapi.v1.Worker.LabelsEntry - (*timestamppb.Timestamp)(nil), // 154: google.protobuf.Timestamp - (*descriptorpb.MessageOptions)(nil), // 155: google.protobuf.MessageOptions - (*descriptorpb.MethodOptions)(nil), // 156: google.protobuf.MethodOptions + (*RuntimeOverhead)(nil), // 117: substrate.ateapi.v1.RuntimeOverhead + (*ActorSandboxRuntime)(nil), // 118: substrate.ateapi.v1.ActorSandboxRuntime + (*VersionedSandboxCompat)(nil), // 119: substrate.ateapi.v1.VersionedSandboxCompat + (*AttributeEntry)(nil), // 120: substrate.ateapi.v1.AttributeEntry + (*WorkerResources)(nil), // 121: substrate.ateapi.v1.WorkerResources + (*ActorAssignment)(nil), // 122: substrate.ateapi.v1.ActorAssignment + (*AcquireWorkerEpochRequest)(nil), // 123: substrate.ateapi.v1.AcquireWorkerEpochRequest + (*AcquireWorkerEpochResponse)(nil), // 124: substrate.ateapi.v1.AcquireWorkerEpochResponse + (*RegisterWorkerRequest)(nil), // 125: substrate.ateapi.v1.RegisterWorkerRequest + (*RegisterWorkerResponse)(nil), // 126: substrate.ateapi.v1.RegisterWorkerResponse + (*SetWorkerSandboxRuntimesRequest)(nil), // 127: substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest + (*SetWorkerSandboxRuntimesResponse)(nil), // 128: substrate.ateapi.v1.SetWorkerSandboxRuntimesResponse + (*MintWorkerActorCertificateRequest)(nil), // 129: substrate.ateapi.v1.MintWorkerActorCertificateRequest + (*MintWorkerActorCertificateResponse)(nil), // 130: substrate.ateapi.v1.MintWorkerActorCertificateResponse + (*MintWorkerActorJwtRequest)(nil), // 131: substrate.ateapi.v1.MintWorkerActorJwtRequest + (*MintWorkerActorJwtResponse)(nil), // 132: substrate.ateapi.v1.MintWorkerActorJwtResponse + (*RequestActorSuspendRequest)(nil), // 133: substrate.ateapi.v1.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 134: substrate.ateapi.v1.RequestActorSuspendResponse + (*ReportActorCrashedRequest)(nil), // 135: substrate.ateapi.v1.ReportActorCrashedRequest + (*ReportActorCrashedResponse)(nil), // 136: substrate.ateapi.v1.ReportActorCrashedResponse + (*ReportActorSnapshotStateRequest)(nil), // 137: substrate.ateapi.v1.ReportActorSnapshotStateRequest + (*ReportActorSnapshotStateResponse)(nil), // 138: substrate.ateapi.v1.ReportActorSnapshotStateResponse + (*ActorSnapshotState)(nil), // 139: substrate.ateapi.v1.ActorSnapshotState + (*SnapshotReport)(nil), // 140: substrate.ateapi.v1.SnapshotReport + (*AccessPolicy)(nil), // 141: substrate.ateapi.v1.AccessPolicy + (*Binding)(nil), // 142: substrate.ateapi.v1.Binding + (*GetGlobalAccessPolicyRequest)(nil), // 143: substrate.ateapi.v1.GetGlobalAccessPolicyRequest + (*CreateGlobalAccessPolicyRequest)(nil), // 144: substrate.ateapi.v1.CreateGlobalAccessPolicyRequest + (*UpdateGlobalAccessPolicyRequest)(nil), // 145: substrate.ateapi.v1.UpdateGlobalAccessPolicyRequest + (*GetAtespaceAccessPolicyRequest)(nil), // 146: substrate.ateapi.v1.GetAtespaceAccessPolicyRequest + (*CreateAtespaceAccessPolicyRequest)(nil), // 147: substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest + (*UpdateAtespaceAccessPolicyRequest)(nil), // 148: substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest + (*DeleteAtespaceAccessPolicyRequest)(nil), // 149: substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest + (*ResourceAnnotation)(nil), // 150: substrate.ateapi.v1.ResourceAnnotation + (*MethodAnnotation)(nil), // 151: substrate.ateapi.v1.MethodAnnotation + nil, // 152: substrate.ateapi.v1.Selector.MatchLabelsEntry + nil, // 153: substrate.ateapi.v1.ExternalVolumeStatus.VolumeContextEntry + nil, // 154: substrate.ateapi.v1.Worker.LabelsEntry + (*timestamppb.Timestamp)(nil), // 155: google.protobuf.Timestamp + (*descriptorpb.MessageOptions)(nil), // 156: google.protobuf.MessageOptions + (*descriptorpb.MethodOptions)(nil), // 157: google.protobuf.MethodOptions } var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ 1, // 0: substrate.ateapi.v1.SnapshotStorage.state:type_name -> substrate.ateapi.v1.SnapshotStorageState @@ -10862,13 +10950,13 @@ var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ 0, // 2: substrate.ateapi.v1.Snapshot.owner:type_name -> substrate.ateapi.v1.SnapshotOwner 2, // 3: substrate.ateapi.v1.Snapshot.fidelity:type_name -> substrate.ateapi.v1.SnapshotFidelity 11, // 4: substrate.ateapi.v1.Snapshot.durable_snapshot:type_name -> substrate.ateapi.v1.SnapshotStorage - 117, // 5: substrate.ateapi.v1.Snapshot.sandbox_runtime:type_name -> substrate.ateapi.v1.ActorSandboxRuntime - 151, // 6: substrate.ateapi.v1.Selector.match_labels:type_name -> substrate.ateapi.v1.Selector.MatchLabelsEntry - 154, // 7: substrate.ateapi.v1.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp - 154, // 8: substrate.ateapi.v1.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp - 154, // 9: substrate.ateapi.v1.ResourceMetadata.delete_time:type_name -> google.protobuf.Timestamp + 118, // 5: substrate.ateapi.v1.Snapshot.sandbox_runtime:type_name -> substrate.ateapi.v1.ActorSandboxRuntime + 152, // 6: substrate.ateapi.v1.Selector.match_labels:type_name -> substrate.ateapi.v1.Selector.MatchLabelsEntry + 155, // 7: substrate.ateapi.v1.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 155, // 8: substrate.ateapi.v1.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 155, // 9: substrate.ateapi.v1.ResourceMetadata.delete_time:type_name -> google.protobuf.Timestamp 16, // 10: substrate.ateapi.v1.ExternalVolumes.volumes:type_name -> substrate.ateapi.v1.ExternalVolumeStatus - 152, // 11: substrate.ateapi.v1.ExternalVolumeStatus.volume_context:type_name -> substrate.ateapi.v1.ExternalVolumeStatus.VolumeContextEntry + 153, // 11: substrate.ateapi.v1.ExternalVolumeStatus.volume_context:type_name -> substrate.ateapi.v1.ExternalVolumeStatus.VolumeContextEntry 14, // 12: substrate.ateapi.v1.Actor.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata 35, // 13: substrate.ateapi.v1.Actor.actor_template:type_name -> substrate.ateapi.v1.ObjectRef 13, // 14: substrate.ateapi.v1.Actor.worker_selector:type_name -> substrate.ateapi.v1.Selector @@ -10893,9 +10981,9 @@ var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ 15, // 33: substrate.ateapi.v1.ActorStatus.external_volumes:type_name -> substrate.ateapi.v1.ExternalVolumes 30, // 34: substrate.ateapi.v1.ActorStatus.crash:type_name -> substrate.ateapi.v1.ActorCrash 12, // 35: substrate.ateapi.v1.ActorStatus.snapshots:type_name -> substrate.ateapi.v1.Snapshot - 154, // 36: substrate.ateapi.v1.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp + 155, // 36: substrate.ateapi.v1.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp 35, // 37: substrate.ateapi.v1.WorkerAssignment.worker:type_name -> substrate.ateapi.v1.ObjectRef - 117, // 38: substrate.ateapi.v1.WorkerAssignment.sandbox_runtime:type_name -> substrate.ateapi.v1.ActorSandboxRuntime + 118, // 38: substrate.ateapi.v1.WorkerAssignment.sandbox_runtime:type_name -> substrate.ateapi.v1.ActorSandboxRuntime 12, // 39: substrate.ateapi.v1.TagStatus.snapshot:type_name -> substrate.ateapi.v1.Snapshot 14, // 40: substrate.ateapi.v1.Tag.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata 32, // 41: substrate.ateapi.v1.Tag.status:type_name -> substrate.ateapi.v1.TagStatus @@ -10912,7 +11000,7 @@ var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ 40, // 52: substrate.ateapi.v1.ActorTemplate.status:type_name -> substrate.ateapi.v1.ActorTemplateStatus 38, // 53: substrate.ateapi.v1.ResourceRequirements.limits:type_name -> substrate.ateapi.v1.Resources 35, // 54: substrate.ateapi.v1.GoldenSnapshotStatus.golden_tag:type_name -> substrate.ateapi.v1.ObjectRef - 154, // 55: substrate.ateapi.v1.GoldenSnapshotStatus.snapshot_time:type_name -> google.protobuf.Timestamp + 155, // 55: substrate.ateapi.v1.GoldenSnapshotStatus.snapshot_time:type_name -> google.protobuf.Timestamp 39, // 56: substrate.ateapi.v1.ActorTemplateStatus.golden_snapshot_status:type_name -> substrate.ateapi.v1.GoldenSnapshotStatus 2, // 57: substrate.ateapi.v1.SnapshotConfig.preferred_fidelity:type_name -> substrate.ateapi.v1.SnapshotFidelity 42, // 58: substrate.ateapi.v1.SnapshotConfig.golden_snapshot_config:type_name -> substrate.ateapi.v1.GoldenSnapshotConfig @@ -10968,7 +11056,7 @@ var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ 95, // 108: substrate.ateapi.v1.DeleteActorEgressPolicyRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions 35, // 109: substrate.ateapi.v1.GetTagRequest.tag:type_name -> substrate.ateapi.v1.ObjectRef 35, // 110: substrate.ateapi.v1.MintActorJwtRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef - 154, // 111: substrate.ateapi.v1.MintActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp + 155, // 111: substrate.ateapi.v1.MintActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp 35, // 112: substrate.ateapi.v1.MintActorCertificateRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef 96, // 113: substrate.ateapi.v1.ListTagsRequest.options:type_name -> substrate.ateapi.v1.ListOptions 33, // 114: substrate.ateapi.v1.ListTagsResponse.tags:type_name -> substrate.ateapi.v1.Tag @@ -10978,7 +11066,7 @@ var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ 95, // 118: substrate.ateapi.v1.DeleteTagRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions 35, // 119: substrate.ateapi.v1.ListWorkerActorAssignmentsRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef 96, // 120: substrate.ateapi.v1.ListWorkerActorAssignmentsRequest.options:type_name -> substrate.ateapi.v1.ListOptions - 121, // 121: substrate.ateapi.v1.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> substrate.ateapi.v1.ActorAssignment + 122, // 121: substrate.ateapi.v1.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> substrate.ateapi.v1.ActorAssignment 96, // 122: substrate.ateapi.v1.ListWorkersRequest.options:type_name -> substrate.ateapi.v1.ListOptions 113, // 123: substrate.ateapi.v1.ListWorkersResponse.workers:type_name -> substrate.ateapi.v1.Worker 35, // 124: substrate.ateapi.v1.GetWorkerRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef @@ -10995,170 +11083,171 @@ var file_api_ateapi_v1_ateapi_proto_depIdxs = []int32{ 96, // 135: substrate.ateapi.v1.ListActorsRequest.options:type_name -> substrate.ateapi.v1.ListOptions 17, // 136: substrate.ateapi.v1.ListActorsResponse.actors:type_name -> substrate.ateapi.v1.Actor 14, // 137: substrate.ateapi.v1.Worker.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata - 153, // 138: substrate.ateapi.v1.Worker.labels:type_name -> substrate.ateapi.v1.Worker.LabelsEntry + 154, // 138: substrate.ateapi.v1.Worker.labels:type_name -> substrate.ateapi.v1.Worker.LabelsEntry 114, // 139: substrate.ateapi.v1.Worker.status:type_name -> substrate.ateapi.v1.WorkerStatus 8, // 140: substrate.ateapi.v1.WorkerStatus.state:type_name -> substrate.ateapi.v1.WorkerState - 120, // 141: substrate.ateapi.v1.WorkerStatus.capacity:type_name -> substrate.ateapi.v1.WorkerResources - 120, // 142: substrate.ateapi.v1.WorkerStatus.allocated:type_name -> substrate.ateapi.v1.WorkerResources + 121, // 141: substrate.ateapi.v1.WorkerStatus.capacity:type_name -> substrate.ateapi.v1.WorkerResources + 121, // 142: substrate.ateapi.v1.WorkerStatus.allocated:type_name -> substrate.ateapi.v1.WorkerResources 115, // 143: substrate.ateapi.v1.WorkerStatus.sandbox_classes:type_name -> substrate.ateapi.v1.WorkerSandboxClass 116, // 144: substrate.ateapi.v1.WorkerSandboxClass.default_runtime:type_name -> substrate.ateapi.v1.WorkerSandboxRuntime 116, // 145: substrate.ateapi.v1.WorkerSandboxClass.restorable_runtimes:type_name -> substrate.ateapi.v1.WorkerSandboxRuntime - 118, // 146: substrate.ateapi.v1.WorkerSandboxRuntime.version:type_name -> substrate.ateapi.v1.VersionedSandboxCompat - 118, // 147: substrate.ateapi.v1.ActorSandboxRuntime.version:type_name -> substrate.ateapi.v1.VersionedSandboxCompat - 119, // 148: substrate.ateapi.v1.VersionedSandboxCompat.attributes:type_name -> substrate.ateapi.v1.AttributeEntry - 14, // 149: substrate.ateapi.v1.ActorAssignment.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata - 35, // 150: substrate.ateapi.v1.ActorAssignment.actor:type_name -> substrate.ateapi.v1.ObjectRef - 38, // 151: substrate.ateapi.v1.ActorAssignment.resources:type_name -> substrate.ateapi.v1.Resources - 35, // 152: substrate.ateapi.v1.AcquireWorkerEpochRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef - 35, // 153: substrate.ateapi.v1.RegisterWorkerRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef - 120, // 154: substrate.ateapi.v1.RegisterWorkerRequest.capacity:type_name -> substrate.ateapi.v1.WorkerResources - 113, // 155: substrate.ateapi.v1.RegisterWorkerResponse.worker:type_name -> substrate.ateapi.v1.Worker - 35, // 156: substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef - 115, // 157: substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest.sandbox_classes:type_name -> substrate.ateapi.v1.WorkerSandboxClass - 113, // 158: substrate.ateapi.v1.SetWorkerSandboxRuntimesResponse.worker:type_name -> substrate.ateapi.v1.Worker - 35, // 159: substrate.ateapi.v1.MintWorkerActorCertificateRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef - 35, // 160: substrate.ateapi.v1.MintWorkerActorCertificateRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef - 35, // 161: substrate.ateapi.v1.MintWorkerActorJwtRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef - 154, // 162: substrate.ateapi.v1.MintWorkerActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp - 35, // 163: substrate.ateapi.v1.RequestActorSuspendRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef - 35, // 164: substrate.ateapi.v1.RequestActorSuspendRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef - 17, // 165: substrate.ateapi.v1.RequestActorSuspendResponse.actor:type_name -> substrate.ateapi.v1.Actor - 35, // 166: substrate.ateapi.v1.ReportActorCrashedRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef - 35, // 167: substrate.ateapi.v1.ReportActorCrashedRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef - 17, // 168: substrate.ateapi.v1.ReportActorCrashedResponse.actor:type_name -> substrate.ateapi.v1.Actor - 35, // 169: substrate.ateapi.v1.ReportActorSnapshotStateRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef - 138, // 170: substrate.ateapi.v1.ReportActorSnapshotStateRequest.state:type_name -> substrate.ateapi.v1.ActorSnapshotState - 139, // 171: substrate.ateapi.v1.ActorSnapshotState.latest_local:type_name -> substrate.ateapi.v1.SnapshotReport - 139, // 172: substrate.ateapi.v1.ActorSnapshotState.uploading:type_name -> substrate.ateapi.v1.SnapshotReport - 139, // 173: substrate.ateapi.v1.ActorSnapshotState.latest_durable:type_name -> substrate.ateapi.v1.SnapshotReport - 154, // 174: substrate.ateapi.v1.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp - 2, // 175: substrate.ateapi.v1.SnapshotReport.fidelity:type_name -> substrate.ateapi.v1.SnapshotFidelity - 10, // 176: substrate.ateapi.v1.SnapshotReport.object:type_name -> substrate.ateapi.v1.ObjectSnapshot - 154, // 177: substrate.ateapi.v1.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp - 154, // 178: substrate.ateapi.v1.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp - 14, // 179: substrate.ateapi.v1.AccessPolicy.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata - 141, // 180: substrate.ateapi.v1.AccessPolicy.bindings:type_name -> substrate.ateapi.v1.Binding - 140, // 181: substrate.ateapi.v1.CreateGlobalAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy - 140, // 182: substrate.ateapi.v1.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy - 35, // 183: substrate.ateapi.v1.GetAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef - 35, // 184: substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef - 140, // 185: substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy - 35, // 186: substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef - 140, // 187: substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy - 35, // 188: substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef - 95, // 189: substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions - 9, // 190: substrate.ateapi.v1.ResourceAnnotation.scope:type_name -> substrate.ateapi.v1.Scope - 155, // 191: substrate.ateapi.v1.resource:extendee -> google.protobuf.MessageOptions - 156, // 192: substrate.ateapi.v1.method:extendee -> google.protobuf.MethodOptions - 149, // 193: substrate.ateapi.v1.resource:type_name -> substrate.ateapi.v1.ResourceAnnotation - 150, // 194: substrate.ateapi.v1.method:type_name -> substrate.ateapi.v1.MethodAnnotation - 70, // 195: substrate.ateapi.v1.Control.GetActor:input_type -> substrate.ateapi.v1.GetActorRequest - 71, // 196: substrate.ateapi.v1.Control.CreateActor:input_type -> substrate.ateapi.v1.CreateActorRequest - 72, // 197: substrate.ateapi.v1.Control.UpdateActor:input_type -> substrate.ateapi.v1.UpdateActorRequest - 73, // 198: substrate.ateapi.v1.Control.SuspendActor:input_type -> substrate.ateapi.v1.SuspendActorRequest - 76, // 199: substrate.ateapi.v1.Control.ResumeActor:input_type -> substrate.ateapi.v1.ResumeActorRequest - 78, // 200: substrate.ateapi.v1.Control.RevertActor:input_type -> substrate.ateapi.v1.RevertActorRequest - 80, // 201: substrate.ateapi.v1.Control.DeleteActor:input_type -> substrate.ateapi.v1.DeleteActorRequest - 81, // 202: substrate.ateapi.v1.Control.GetActorEgressPolicy:input_type -> substrate.ateapi.v1.GetActorEgressPolicyRequest - 82, // 203: substrate.ateapi.v1.Control.CreateActorEgressPolicy:input_type -> substrate.ateapi.v1.CreateActorEgressPolicyRequest - 83, // 204: substrate.ateapi.v1.Control.UpdateActorEgressPolicy:input_type -> substrate.ateapi.v1.UpdateActorEgressPolicyRequest - 84, // 205: substrate.ateapi.v1.Control.DeleteActorEgressPolicy:input_type -> substrate.ateapi.v1.DeleteActorEgressPolicyRequest - 86, // 206: substrate.ateapi.v1.Control.MintActorJwt:input_type -> substrate.ateapi.v1.MintActorJwtRequest - 88, // 207: substrate.ateapi.v1.Control.MintActorCertificate:input_type -> substrate.ateapi.v1.MintActorCertificateRequest - 92, // 208: substrate.ateapi.v1.Control.CreateTag:input_type -> substrate.ateapi.v1.CreateTagRequest - 85, // 209: substrate.ateapi.v1.Control.GetTag:input_type -> substrate.ateapi.v1.GetTagRequest - 90, // 210: substrate.ateapi.v1.Control.ListTags:input_type -> substrate.ateapi.v1.ListTagsRequest - 93, // 211: substrate.ateapi.v1.Control.UpdateTag:input_type -> substrate.ateapi.v1.UpdateTagRequest - 94, // 212: substrate.ateapi.v1.Control.DeleteTag:input_type -> substrate.ateapi.v1.DeleteTagRequest - 99, // 213: substrate.ateapi.v1.Control.ListWorkers:input_type -> substrate.ateapi.v1.ListWorkersRequest - 101, // 214: substrate.ateapi.v1.Control.GetWorker:input_type -> substrate.ateapi.v1.GetWorkerRequest - 102, // 215: substrate.ateapi.v1.Control.CreateWorker:input_type -> substrate.ateapi.v1.CreateWorkerRequest - 103, // 216: substrate.ateapi.v1.Control.UpdateWorker:input_type -> substrate.ateapi.v1.UpdateWorkerRequest - 104, // 217: substrate.ateapi.v1.Control.DeleteWorker:input_type -> substrate.ateapi.v1.DeleteWorkerRequest - 105, // 218: substrate.ateapi.v1.Control.DrainWorker:input_type -> substrate.ateapi.v1.DrainWorkerRequest - 107, // 219: substrate.ateapi.v1.Control.MarkWorkerAvailable:input_type -> substrate.ateapi.v1.MarkWorkerAvailableRequest - 109, // 220: substrate.ateapi.v1.Control.MarkWorkerUnavailable:input_type -> substrate.ateapi.v1.MarkWorkerUnavailableRequest - 97, // 221: substrate.ateapi.v1.Control.ListWorkerActorAssignments:input_type -> substrate.ateapi.v1.ListWorkerActorAssignmentsRequest - 111, // 222: substrate.ateapi.v1.Control.ListActors:input_type -> substrate.ateapi.v1.ListActorsRequest - 60, // 223: substrate.ateapi.v1.Control.CreateAtespace:input_type -> substrate.ateapi.v1.CreateAtespaceRequest - 61, // 224: substrate.ateapi.v1.Control.GetAtespace:input_type -> substrate.ateapi.v1.GetAtespaceRequest - 62, // 225: substrate.ateapi.v1.Control.ListAtespaces:input_type -> substrate.ateapi.v1.ListAtespacesRequest - 64, // 226: substrate.ateapi.v1.Control.DeleteAtespace:input_type -> substrate.ateapi.v1.DeleteAtespaceRequest - 65, // 227: substrate.ateapi.v1.Control.CreateActorTemplate:input_type -> substrate.ateapi.v1.CreateActorTemplateRequest - 66, // 228: substrate.ateapi.v1.Control.GetActorTemplate:input_type -> substrate.ateapi.v1.GetActorTemplateRequest - 67, // 229: substrate.ateapi.v1.Control.ListActorTemplates:input_type -> substrate.ateapi.v1.ListActorTemplatesRequest - 69, // 230: substrate.ateapi.v1.Control.DeleteActorTemplate:input_type -> substrate.ateapi.v1.DeleteActorTemplateRequest - 142, // 231: substrate.ateapi.v1.Control.GetGlobalAccessPolicy:input_type -> substrate.ateapi.v1.GetGlobalAccessPolicyRequest - 143, // 232: substrate.ateapi.v1.Control.CreateGlobalAccessPolicy:input_type -> substrate.ateapi.v1.CreateGlobalAccessPolicyRequest - 144, // 233: substrate.ateapi.v1.Control.UpdateGlobalAccessPolicy:input_type -> substrate.ateapi.v1.UpdateGlobalAccessPolicyRequest - 145, // 234: substrate.ateapi.v1.Control.GetAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.GetAtespaceAccessPolicyRequest - 146, // 235: substrate.ateapi.v1.Control.CreateAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest - 147, // 236: substrate.ateapi.v1.Control.UpdateAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest - 148, // 237: substrate.ateapi.v1.Control.DeleteAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest - 122, // 238: substrate.ateapi.v1.WorkerService.AcquireWorkerEpoch:input_type -> substrate.ateapi.v1.AcquireWorkerEpochRequest - 124, // 239: substrate.ateapi.v1.WorkerService.RegisterWorker:input_type -> substrate.ateapi.v1.RegisterWorkerRequest - 126, // 240: substrate.ateapi.v1.WorkerService.SetWorkerSandboxRuntimes:input_type -> substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest - 128, // 241: substrate.ateapi.v1.WorkerService.MintWorkerActorCertificate:input_type -> substrate.ateapi.v1.MintWorkerActorCertificateRequest - 130, // 242: substrate.ateapi.v1.WorkerService.MintWorkerActorJwt:input_type -> substrate.ateapi.v1.MintWorkerActorJwtRequest - 132, // 243: substrate.ateapi.v1.WorkerService.RequestActorSuspend:input_type -> substrate.ateapi.v1.RequestActorSuspendRequest - 136, // 244: substrate.ateapi.v1.WorkerService.ReportActorSnapshotState:input_type -> substrate.ateapi.v1.ReportActorSnapshotStateRequest - 134, // 245: substrate.ateapi.v1.WorkerService.ReportActorCrashed:input_type -> substrate.ateapi.v1.ReportActorCrashedRequest - 17, // 246: substrate.ateapi.v1.Control.GetActor:output_type -> substrate.ateapi.v1.Actor - 17, // 247: substrate.ateapi.v1.Control.CreateActor:output_type -> substrate.ateapi.v1.Actor - 17, // 248: substrate.ateapi.v1.Control.UpdateActor:output_type -> substrate.ateapi.v1.Actor - 74, // 249: substrate.ateapi.v1.Control.SuspendActor:output_type -> substrate.ateapi.v1.SuspendActorResponse - 77, // 250: substrate.ateapi.v1.Control.ResumeActor:output_type -> substrate.ateapi.v1.ResumeActorResponse - 79, // 251: substrate.ateapi.v1.Control.RevertActor:output_type -> substrate.ateapi.v1.RevertActorResponse - 17, // 252: substrate.ateapi.v1.Control.DeleteActor:output_type -> substrate.ateapi.v1.Actor - 18, // 253: substrate.ateapi.v1.Control.GetActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy - 18, // 254: substrate.ateapi.v1.Control.CreateActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy - 18, // 255: substrate.ateapi.v1.Control.UpdateActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy - 18, // 256: substrate.ateapi.v1.Control.DeleteActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy - 87, // 257: substrate.ateapi.v1.Control.MintActorJwt:output_type -> substrate.ateapi.v1.MintActorJwtResponse - 89, // 258: substrate.ateapi.v1.Control.MintActorCertificate:output_type -> substrate.ateapi.v1.MintActorCertificateResponse - 33, // 259: substrate.ateapi.v1.Control.CreateTag:output_type -> substrate.ateapi.v1.Tag - 33, // 260: substrate.ateapi.v1.Control.GetTag:output_type -> substrate.ateapi.v1.Tag - 91, // 261: substrate.ateapi.v1.Control.ListTags:output_type -> substrate.ateapi.v1.ListTagsResponse - 33, // 262: substrate.ateapi.v1.Control.UpdateTag:output_type -> substrate.ateapi.v1.Tag - 33, // 263: substrate.ateapi.v1.Control.DeleteTag:output_type -> substrate.ateapi.v1.Tag - 100, // 264: substrate.ateapi.v1.Control.ListWorkers:output_type -> substrate.ateapi.v1.ListWorkersResponse - 113, // 265: substrate.ateapi.v1.Control.GetWorker:output_type -> substrate.ateapi.v1.Worker - 113, // 266: substrate.ateapi.v1.Control.CreateWorker:output_type -> substrate.ateapi.v1.Worker - 113, // 267: substrate.ateapi.v1.Control.UpdateWorker:output_type -> substrate.ateapi.v1.Worker - 113, // 268: substrate.ateapi.v1.Control.DeleteWorker:output_type -> substrate.ateapi.v1.Worker - 106, // 269: substrate.ateapi.v1.Control.DrainWorker:output_type -> substrate.ateapi.v1.DrainWorkerResponse - 108, // 270: substrate.ateapi.v1.Control.MarkWorkerAvailable:output_type -> substrate.ateapi.v1.MarkWorkerAvailableResponse - 110, // 271: substrate.ateapi.v1.Control.MarkWorkerUnavailable:output_type -> substrate.ateapi.v1.MarkWorkerUnavailableResponse - 98, // 272: substrate.ateapi.v1.Control.ListWorkerActorAssignments:output_type -> substrate.ateapi.v1.ListWorkerActorAssignmentsResponse - 112, // 273: substrate.ateapi.v1.Control.ListActors:output_type -> substrate.ateapi.v1.ListActorsResponse - 34, // 274: substrate.ateapi.v1.Control.CreateAtespace:output_type -> substrate.ateapi.v1.Atespace - 34, // 275: substrate.ateapi.v1.Control.GetAtespace:output_type -> substrate.ateapi.v1.Atespace - 63, // 276: substrate.ateapi.v1.Control.ListAtespaces:output_type -> substrate.ateapi.v1.ListAtespacesResponse - 34, // 277: substrate.ateapi.v1.Control.DeleteAtespace:output_type -> substrate.ateapi.v1.Atespace - 36, // 278: substrate.ateapi.v1.Control.CreateActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate - 36, // 279: substrate.ateapi.v1.Control.GetActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate - 68, // 280: substrate.ateapi.v1.Control.ListActorTemplates:output_type -> substrate.ateapi.v1.ListActorTemplatesResponse - 36, // 281: substrate.ateapi.v1.Control.DeleteActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate - 140, // 282: substrate.ateapi.v1.Control.GetGlobalAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 140, // 283: substrate.ateapi.v1.Control.CreateGlobalAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 140, // 284: substrate.ateapi.v1.Control.UpdateGlobalAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 140, // 285: substrate.ateapi.v1.Control.GetAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 140, // 286: substrate.ateapi.v1.Control.CreateAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 140, // 287: substrate.ateapi.v1.Control.UpdateAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 140, // 288: substrate.ateapi.v1.Control.DeleteAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy - 123, // 289: substrate.ateapi.v1.WorkerService.AcquireWorkerEpoch:output_type -> substrate.ateapi.v1.AcquireWorkerEpochResponse - 125, // 290: substrate.ateapi.v1.WorkerService.RegisterWorker:output_type -> substrate.ateapi.v1.RegisterWorkerResponse - 127, // 291: substrate.ateapi.v1.WorkerService.SetWorkerSandboxRuntimes:output_type -> substrate.ateapi.v1.SetWorkerSandboxRuntimesResponse - 129, // 292: substrate.ateapi.v1.WorkerService.MintWorkerActorCertificate:output_type -> substrate.ateapi.v1.MintWorkerActorCertificateResponse - 131, // 293: substrate.ateapi.v1.WorkerService.MintWorkerActorJwt:output_type -> substrate.ateapi.v1.MintWorkerActorJwtResponse - 133, // 294: substrate.ateapi.v1.WorkerService.RequestActorSuspend:output_type -> substrate.ateapi.v1.RequestActorSuspendResponse - 137, // 295: substrate.ateapi.v1.WorkerService.ReportActorSnapshotState:output_type -> substrate.ateapi.v1.ReportActorSnapshotStateResponse - 135, // 296: substrate.ateapi.v1.WorkerService.ReportActorCrashed:output_type -> substrate.ateapi.v1.ReportActorCrashedResponse - 246, // [246:297] is the sub-list for method output_type - 195, // [195:246] is the sub-list for method input_type - 193, // [193:195] is the sub-list for extension type_name - 191, // [191:193] is the sub-list for extension extendee - 0, // [0:191] is the sub-list for field type_name + 119, // 146: substrate.ateapi.v1.WorkerSandboxRuntime.version:type_name -> substrate.ateapi.v1.VersionedSandboxCompat + 117, // 147: substrate.ateapi.v1.WorkerSandboxRuntime.actor_overhead:type_name -> substrate.ateapi.v1.RuntimeOverhead + 119, // 148: substrate.ateapi.v1.ActorSandboxRuntime.version:type_name -> substrate.ateapi.v1.VersionedSandboxCompat + 120, // 149: substrate.ateapi.v1.VersionedSandboxCompat.attributes:type_name -> substrate.ateapi.v1.AttributeEntry + 14, // 150: substrate.ateapi.v1.ActorAssignment.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 35, // 151: substrate.ateapi.v1.ActorAssignment.actor:type_name -> substrate.ateapi.v1.ObjectRef + 38, // 152: substrate.ateapi.v1.ActorAssignment.resources:type_name -> substrate.ateapi.v1.Resources + 35, // 153: substrate.ateapi.v1.AcquireWorkerEpochRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef + 35, // 154: substrate.ateapi.v1.RegisterWorkerRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef + 121, // 155: substrate.ateapi.v1.RegisterWorkerRequest.capacity:type_name -> substrate.ateapi.v1.WorkerResources + 113, // 156: substrate.ateapi.v1.RegisterWorkerResponse.worker:type_name -> substrate.ateapi.v1.Worker + 35, // 157: substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef + 115, // 158: substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest.sandbox_classes:type_name -> substrate.ateapi.v1.WorkerSandboxClass + 113, // 159: substrate.ateapi.v1.SetWorkerSandboxRuntimesResponse.worker:type_name -> substrate.ateapi.v1.Worker + 35, // 160: substrate.ateapi.v1.MintWorkerActorCertificateRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 35, // 161: substrate.ateapi.v1.MintWorkerActorCertificateRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef + 35, // 162: substrate.ateapi.v1.MintWorkerActorJwtRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 155, // 163: substrate.ateapi.v1.MintWorkerActorJwtResponse.expire_time:type_name -> google.protobuf.Timestamp + 35, // 164: substrate.ateapi.v1.RequestActorSuspendRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef + 35, // 165: substrate.ateapi.v1.RequestActorSuspendRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 17, // 166: substrate.ateapi.v1.RequestActorSuspendResponse.actor:type_name -> substrate.ateapi.v1.Actor + 35, // 167: substrate.ateapi.v1.ReportActorCrashedRequest.worker:type_name -> substrate.ateapi.v1.ObjectRef + 35, // 168: substrate.ateapi.v1.ReportActorCrashedRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 17, // 169: substrate.ateapi.v1.ReportActorCrashedResponse.actor:type_name -> substrate.ateapi.v1.Actor + 35, // 170: substrate.ateapi.v1.ReportActorSnapshotStateRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 139, // 171: substrate.ateapi.v1.ReportActorSnapshotStateRequest.state:type_name -> substrate.ateapi.v1.ActorSnapshotState + 140, // 172: substrate.ateapi.v1.ActorSnapshotState.latest_local:type_name -> substrate.ateapi.v1.SnapshotReport + 140, // 173: substrate.ateapi.v1.ActorSnapshotState.uploading:type_name -> substrate.ateapi.v1.SnapshotReport + 140, // 174: substrate.ateapi.v1.ActorSnapshotState.latest_durable:type_name -> substrate.ateapi.v1.SnapshotReport + 155, // 175: substrate.ateapi.v1.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp + 2, // 176: substrate.ateapi.v1.SnapshotReport.fidelity:type_name -> substrate.ateapi.v1.SnapshotFidelity + 10, // 177: substrate.ateapi.v1.SnapshotReport.object:type_name -> substrate.ateapi.v1.ObjectSnapshot + 155, // 178: substrate.ateapi.v1.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp + 155, // 179: substrate.ateapi.v1.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp + 14, // 180: substrate.ateapi.v1.AccessPolicy.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 142, // 181: substrate.ateapi.v1.AccessPolicy.bindings:type_name -> substrate.ateapi.v1.Binding + 141, // 182: substrate.ateapi.v1.CreateGlobalAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy + 141, // 183: substrate.ateapi.v1.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy + 35, // 184: substrate.ateapi.v1.GetAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef + 35, // 185: substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef + 141, // 186: substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy + 35, // 187: substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef + 141, // 188: substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> substrate.ateapi.v1.AccessPolicy + 35, // 189: substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> substrate.ateapi.v1.ObjectRef + 95, // 190: substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions + 9, // 191: substrate.ateapi.v1.ResourceAnnotation.scope:type_name -> substrate.ateapi.v1.Scope + 156, // 192: substrate.ateapi.v1.resource:extendee -> google.protobuf.MessageOptions + 157, // 193: substrate.ateapi.v1.method:extendee -> google.protobuf.MethodOptions + 150, // 194: substrate.ateapi.v1.resource:type_name -> substrate.ateapi.v1.ResourceAnnotation + 151, // 195: substrate.ateapi.v1.method:type_name -> substrate.ateapi.v1.MethodAnnotation + 70, // 196: substrate.ateapi.v1.Control.GetActor:input_type -> substrate.ateapi.v1.GetActorRequest + 71, // 197: substrate.ateapi.v1.Control.CreateActor:input_type -> substrate.ateapi.v1.CreateActorRequest + 72, // 198: substrate.ateapi.v1.Control.UpdateActor:input_type -> substrate.ateapi.v1.UpdateActorRequest + 73, // 199: substrate.ateapi.v1.Control.SuspendActor:input_type -> substrate.ateapi.v1.SuspendActorRequest + 76, // 200: substrate.ateapi.v1.Control.ResumeActor:input_type -> substrate.ateapi.v1.ResumeActorRequest + 78, // 201: substrate.ateapi.v1.Control.RevertActor:input_type -> substrate.ateapi.v1.RevertActorRequest + 80, // 202: substrate.ateapi.v1.Control.DeleteActor:input_type -> substrate.ateapi.v1.DeleteActorRequest + 81, // 203: substrate.ateapi.v1.Control.GetActorEgressPolicy:input_type -> substrate.ateapi.v1.GetActorEgressPolicyRequest + 82, // 204: substrate.ateapi.v1.Control.CreateActorEgressPolicy:input_type -> substrate.ateapi.v1.CreateActorEgressPolicyRequest + 83, // 205: substrate.ateapi.v1.Control.UpdateActorEgressPolicy:input_type -> substrate.ateapi.v1.UpdateActorEgressPolicyRequest + 84, // 206: substrate.ateapi.v1.Control.DeleteActorEgressPolicy:input_type -> substrate.ateapi.v1.DeleteActorEgressPolicyRequest + 86, // 207: substrate.ateapi.v1.Control.MintActorJwt:input_type -> substrate.ateapi.v1.MintActorJwtRequest + 88, // 208: substrate.ateapi.v1.Control.MintActorCertificate:input_type -> substrate.ateapi.v1.MintActorCertificateRequest + 92, // 209: substrate.ateapi.v1.Control.CreateTag:input_type -> substrate.ateapi.v1.CreateTagRequest + 85, // 210: substrate.ateapi.v1.Control.GetTag:input_type -> substrate.ateapi.v1.GetTagRequest + 90, // 211: substrate.ateapi.v1.Control.ListTags:input_type -> substrate.ateapi.v1.ListTagsRequest + 93, // 212: substrate.ateapi.v1.Control.UpdateTag:input_type -> substrate.ateapi.v1.UpdateTagRequest + 94, // 213: substrate.ateapi.v1.Control.DeleteTag:input_type -> substrate.ateapi.v1.DeleteTagRequest + 99, // 214: substrate.ateapi.v1.Control.ListWorkers:input_type -> substrate.ateapi.v1.ListWorkersRequest + 101, // 215: substrate.ateapi.v1.Control.GetWorker:input_type -> substrate.ateapi.v1.GetWorkerRequest + 102, // 216: substrate.ateapi.v1.Control.CreateWorker:input_type -> substrate.ateapi.v1.CreateWorkerRequest + 103, // 217: substrate.ateapi.v1.Control.UpdateWorker:input_type -> substrate.ateapi.v1.UpdateWorkerRequest + 104, // 218: substrate.ateapi.v1.Control.DeleteWorker:input_type -> substrate.ateapi.v1.DeleteWorkerRequest + 105, // 219: substrate.ateapi.v1.Control.DrainWorker:input_type -> substrate.ateapi.v1.DrainWorkerRequest + 107, // 220: substrate.ateapi.v1.Control.MarkWorkerAvailable:input_type -> substrate.ateapi.v1.MarkWorkerAvailableRequest + 109, // 221: substrate.ateapi.v1.Control.MarkWorkerUnavailable:input_type -> substrate.ateapi.v1.MarkWorkerUnavailableRequest + 97, // 222: substrate.ateapi.v1.Control.ListWorkerActorAssignments:input_type -> substrate.ateapi.v1.ListWorkerActorAssignmentsRequest + 111, // 223: substrate.ateapi.v1.Control.ListActors:input_type -> substrate.ateapi.v1.ListActorsRequest + 60, // 224: substrate.ateapi.v1.Control.CreateAtespace:input_type -> substrate.ateapi.v1.CreateAtespaceRequest + 61, // 225: substrate.ateapi.v1.Control.GetAtespace:input_type -> substrate.ateapi.v1.GetAtespaceRequest + 62, // 226: substrate.ateapi.v1.Control.ListAtespaces:input_type -> substrate.ateapi.v1.ListAtespacesRequest + 64, // 227: substrate.ateapi.v1.Control.DeleteAtespace:input_type -> substrate.ateapi.v1.DeleteAtespaceRequest + 65, // 228: substrate.ateapi.v1.Control.CreateActorTemplate:input_type -> substrate.ateapi.v1.CreateActorTemplateRequest + 66, // 229: substrate.ateapi.v1.Control.GetActorTemplate:input_type -> substrate.ateapi.v1.GetActorTemplateRequest + 67, // 230: substrate.ateapi.v1.Control.ListActorTemplates:input_type -> substrate.ateapi.v1.ListActorTemplatesRequest + 69, // 231: substrate.ateapi.v1.Control.DeleteActorTemplate:input_type -> substrate.ateapi.v1.DeleteActorTemplateRequest + 143, // 232: substrate.ateapi.v1.Control.GetGlobalAccessPolicy:input_type -> substrate.ateapi.v1.GetGlobalAccessPolicyRequest + 144, // 233: substrate.ateapi.v1.Control.CreateGlobalAccessPolicy:input_type -> substrate.ateapi.v1.CreateGlobalAccessPolicyRequest + 145, // 234: substrate.ateapi.v1.Control.UpdateGlobalAccessPolicy:input_type -> substrate.ateapi.v1.UpdateGlobalAccessPolicyRequest + 146, // 235: substrate.ateapi.v1.Control.GetAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.GetAtespaceAccessPolicyRequest + 147, // 236: substrate.ateapi.v1.Control.CreateAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.CreateAtespaceAccessPolicyRequest + 148, // 237: substrate.ateapi.v1.Control.UpdateAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.UpdateAtespaceAccessPolicyRequest + 149, // 238: substrate.ateapi.v1.Control.DeleteAtespaceAccessPolicy:input_type -> substrate.ateapi.v1.DeleteAtespaceAccessPolicyRequest + 123, // 239: substrate.ateapi.v1.WorkerService.AcquireWorkerEpoch:input_type -> substrate.ateapi.v1.AcquireWorkerEpochRequest + 125, // 240: substrate.ateapi.v1.WorkerService.RegisterWorker:input_type -> substrate.ateapi.v1.RegisterWorkerRequest + 127, // 241: substrate.ateapi.v1.WorkerService.SetWorkerSandboxRuntimes:input_type -> substrate.ateapi.v1.SetWorkerSandboxRuntimesRequest + 129, // 242: substrate.ateapi.v1.WorkerService.MintWorkerActorCertificate:input_type -> substrate.ateapi.v1.MintWorkerActorCertificateRequest + 131, // 243: substrate.ateapi.v1.WorkerService.MintWorkerActorJwt:input_type -> substrate.ateapi.v1.MintWorkerActorJwtRequest + 133, // 244: substrate.ateapi.v1.WorkerService.RequestActorSuspend:input_type -> substrate.ateapi.v1.RequestActorSuspendRequest + 137, // 245: substrate.ateapi.v1.WorkerService.ReportActorSnapshotState:input_type -> substrate.ateapi.v1.ReportActorSnapshotStateRequest + 135, // 246: substrate.ateapi.v1.WorkerService.ReportActorCrashed:input_type -> substrate.ateapi.v1.ReportActorCrashedRequest + 17, // 247: substrate.ateapi.v1.Control.GetActor:output_type -> substrate.ateapi.v1.Actor + 17, // 248: substrate.ateapi.v1.Control.CreateActor:output_type -> substrate.ateapi.v1.Actor + 17, // 249: substrate.ateapi.v1.Control.UpdateActor:output_type -> substrate.ateapi.v1.Actor + 74, // 250: substrate.ateapi.v1.Control.SuspendActor:output_type -> substrate.ateapi.v1.SuspendActorResponse + 77, // 251: substrate.ateapi.v1.Control.ResumeActor:output_type -> substrate.ateapi.v1.ResumeActorResponse + 79, // 252: substrate.ateapi.v1.Control.RevertActor:output_type -> substrate.ateapi.v1.RevertActorResponse + 17, // 253: substrate.ateapi.v1.Control.DeleteActor:output_type -> substrate.ateapi.v1.Actor + 18, // 254: substrate.ateapi.v1.Control.GetActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy + 18, // 255: substrate.ateapi.v1.Control.CreateActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy + 18, // 256: substrate.ateapi.v1.Control.UpdateActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy + 18, // 257: substrate.ateapi.v1.Control.DeleteActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy + 87, // 258: substrate.ateapi.v1.Control.MintActorJwt:output_type -> substrate.ateapi.v1.MintActorJwtResponse + 89, // 259: substrate.ateapi.v1.Control.MintActorCertificate:output_type -> substrate.ateapi.v1.MintActorCertificateResponse + 33, // 260: substrate.ateapi.v1.Control.CreateTag:output_type -> substrate.ateapi.v1.Tag + 33, // 261: substrate.ateapi.v1.Control.GetTag:output_type -> substrate.ateapi.v1.Tag + 91, // 262: substrate.ateapi.v1.Control.ListTags:output_type -> substrate.ateapi.v1.ListTagsResponse + 33, // 263: substrate.ateapi.v1.Control.UpdateTag:output_type -> substrate.ateapi.v1.Tag + 33, // 264: substrate.ateapi.v1.Control.DeleteTag:output_type -> substrate.ateapi.v1.Tag + 100, // 265: substrate.ateapi.v1.Control.ListWorkers:output_type -> substrate.ateapi.v1.ListWorkersResponse + 113, // 266: substrate.ateapi.v1.Control.GetWorker:output_type -> substrate.ateapi.v1.Worker + 113, // 267: substrate.ateapi.v1.Control.CreateWorker:output_type -> substrate.ateapi.v1.Worker + 113, // 268: substrate.ateapi.v1.Control.UpdateWorker:output_type -> substrate.ateapi.v1.Worker + 113, // 269: substrate.ateapi.v1.Control.DeleteWorker:output_type -> substrate.ateapi.v1.Worker + 106, // 270: substrate.ateapi.v1.Control.DrainWorker:output_type -> substrate.ateapi.v1.DrainWorkerResponse + 108, // 271: substrate.ateapi.v1.Control.MarkWorkerAvailable:output_type -> substrate.ateapi.v1.MarkWorkerAvailableResponse + 110, // 272: substrate.ateapi.v1.Control.MarkWorkerUnavailable:output_type -> substrate.ateapi.v1.MarkWorkerUnavailableResponse + 98, // 273: substrate.ateapi.v1.Control.ListWorkerActorAssignments:output_type -> substrate.ateapi.v1.ListWorkerActorAssignmentsResponse + 112, // 274: substrate.ateapi.v1.Control.ListActors:output_type -> substrate.ateapi.v1.ListActorsResponse + 34, // 275: substrate.ateapi.v1.Control.CreateAtespace:output_type -> substrate.ateapi.v1.Atespace + 34, // 276: substrate.ateapi.v1.Control.GetAtespace:output_type -> substrate.ateapi.v1.Atespace + 63, // 277: substrate.ateapi.v1.Control.ListAtespaces:output_type -> substrate.ateapi.v1.ListAtespacesResponse + 34, // 278: substrate.ateapi.v1.Control.DeleteAtespace:output_type -> substrate.ateapi.v1.Atespace + 36, // 279: substrate.ateapi.v1.Control.CreateActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 36, // 280: substrate.ateapi.v1.Control.GetActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 68, // 281: substrate.ateapi.v1.Control.ListActorTemplates:output_type -> substrate.ateapi.v1.ListActorTemplatesResponse + 36, // 282: substrate.ateapi.v1.Control.DeleteActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 141, // 283: substrate.ateapi.v1.Control.GetGlobalAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 141, // 284: substrate.ateapi.v1.Control.CreateGlobalAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 141, // 285: substrate.ateapi.v1.Control.UpdateGlobalAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 141, // 286: substrate.ateapi.v1.Control.GetAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 141, // 287: substrate.ateapi.v1.Control.CreateAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 141, // 288: substrate.ateapi.v1.Control.UpdateAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 141, // 289: substrate.ateapi.v1.Control.DeleteAtespaceAccessPolicy:output_type -> substrate.ateapi.v1.AccessPolicy + 124, // 290: substrate.ateapi.v1.WorkerService.AcquireWorkerEpoch:output_type -> substrate.ateapi.v1.AcquireWorkerEpochResponse + 126, // 291: substrate.ateapi.v1.WorkerService.RegisterWorker:output_type -> substrate.ateapi.v1.RegisterWorkerResponse + 128, // 292: substrate.ateapi.v1.WorkerService.SetWorkerSandboxRuntimes:output_type -> substrate.ateapi.v1.SetWorkerSandboxRuntimesResponse + 130, // 293: substrate.ateapi.v1.WorkerService.MintWorkerActorCertificate:output_type -> substrate.ateapi.v1.MintWorkerActorCertificateResponse + 132, // 294: substrate.ateapi.v1.WorkerService.MintWorkerActorJwt:output_type -> substrate.ateapi.v1.MintWorkerActorJwtResponse + 134, // 295: substrate.ateapi.v1.WorkerService.RequestActorSuspend:output_type -> substrate.ateapi.v1.RequestActorSuspendResponse + 138, // 296: substrate.ateapi.v1.WorkerService.ReportActorSnapshotState:output_type -> substrate.ateapi.v1.ReportActorSnapshotStateResponse + 136, // 297: substrate.ateapi.v1.WorkerService.ReportActorCrashed:output_type -> substrate.ateapi.v1.ReportActorCrashedResponse + 247, // [247:298] is the sub-list for method output_type + 196, // [196:247] is the sub-list for method input_type + 194, // [194:196] is the sub-list for extension type_name + 192, // [192:194] is the sub-list for extension extendee + 0, // [0:192] is the sub-list for field type_name } func init() { file_api_ateapi_v1_ateapi_proto_init() } @@ -11172,7 +11261,7 @@ func file_api_ateapi_v1_ateapi_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_api_ateapi_v1_ateapi_proto_rawDesc), len(file_api_ateapi_v1_ateapi_proto_rawDesc)), NumEnums: 10, - NumMessages: 144, + NumMessages: 145, NumExtensions: 2, NumServices: 2, }, diff --git a/api/ateapi/v1/ateapi.proto b/api/ateapi/v1/ateapi.proto index d2b365c2b3..fb6c701357 100644 --- a/api/ateapi/v1/ateapi.proto +++ b/api/ateapi/v1/ateapi.proto @@ -2409,7 +2409,8 @@ message WorkerStatus { WorkerResources capacity = 2; // The allocated resources of the Worker. This is the sum of all the - // resources consumed by the Actors currently assigned to this Worker. + // resources consumed by the Actors currently assigned to this Worker, + // including their runtime's actor_overhead. // // +k8s:optional WorkerResources allocated = 3; @@ -2505,10 +2506,39 @@ message WorkerSandboxRuntime { // +k8s:required VersionedSandboxCompat version = 1; + // The resource overhead incurred by a Worker for each Actor running under + // this runtime, on top of the Actor's stated limits. Placement considers + // this when deciding if a Worker has enough capacity for an Actor, and it is + // accounted for in allocated. If not specified, Actors under this runtime + // are assumed to have negligible overhead. + // + // +k8s:optional + RuntimeOverhead actor_overhead = 2; + // TODO: Add an informational SandboxConfig version name (e.g. "gvisor-2") // once workers report one. } +// RuntimeOverhead is the memory an Actor uses on its Worker outside its own +// limits: a fixed amount, plus an amount in proportion to its memory limit. +message RuntimeOverhead { + // memory_bytes is a fixed amount of memory for each Actor. + // + // +k8s:optional + // +k8s:minimum=0 + // +k8s:maximum=1099511627776 # 1TiB + int64 memory_bytes = 1; + + // memory_bytes_per_gib is memory per GiB of the Actor's memory limit, e.g. + // host page tables for a microVM's guest RAM. It scales in proportion and + // rounds down to a byte, so an Actor with a 512MiB limit adds half of it. + // + // +k8s:optional + // +k8s:minimum=0 + // +k8s:maximum=1073741824 # 1GiB + int64 memory_bytes_per_gib = 2; +} + // ActorSandboxRuntime is the runtime an actor runs under, as recorded on its // assignment and on the snapshots taken from it: one class and one of the // compatibility versions a Worker reports for it. @@ -2630,7 +2660,8 @@ message ActorAssignment { // +k8s:format=k8s-uuid string actor_uid = 3; - // The resources allocated to the Actor when it was assigned. + // The resources allocated to the Actor when it was assigned: its limits, + // plus the actor_overhead of the runtime it runs under on the Worker. // // +k8s:optional Resources resources = 4; diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index 76f3272e28..f60e90e455 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -451,8 +451,8 @@ def deploy_workloads( "--sandbox-class", sandbox_class, ] - # Empty keeps the default in workloads/deploy.sh (256Mi, the microvm - # minimum); RAM-consuming suites set actorMemory in tests.yaml. + # Empty keeps the default in workloads/deploy.sh (256Mi); RAM-consuming + # suites set actorMemory in tests.yaml. if actor_memory: cmd += ["--actor-memory", actor_memory] # Empty keeps the default in workloads/deploy.sh (1 CPU). The scheduler diff --git a/benchmarking/automation/tests.yaml b/benchmarking/automation/tests.yaml index 678cbf6ec5..42e13cff07 100644 --- a/benchmarking/automation/tests.yaml +++ b/benchmarking/automation/tests.yaml @@ -32,8 +32,8 @@ # deploy that runs before this test. The orchestrator # redeploys workloads between tests, so each test can # pick its own scale. -# actorMemory Optional (default "256Mi", the smallest size microvm -# admits). Memory limit on the benchmark ActorTemplates; +# actorMemory Optional (default "256Mi"). Memory limit on the +# benchmark ActorTemplates; # raise it for suites that exercise glutton's WriteRAM path. # actorCPU Optional (default "1"). CPU limit on the benchmark # ActorTemplates. The scheduler books it against each @@ -43,8 +43,8 @@ # or limits). Memory request and limit on each WorkerPool # pod. Use it to fix how many workers share a node: one per # node needs more than half the node's allocatable memory. -# The limit is also the memory the worker reports as its -# actor capacity. +# The limit, less what the worker keeps for itself, is +# the memory it reports as its actor capacity. # workerWaitTimeout # Optional (default 300, set by workloads/deploy.sh). # Whole seconds to wait after the WorkerPool deploy for @@ -174,9 +174,9 @@ tests: # GluttonUser fills the actor's RAM to --mem-target through the # glutton WriteRAM API before the first suspend, so every suspend/resume # cycle is measured against a realistically-sized actor. actorMemory = - # target + headroom (glutton itself, the guest, and on microvm the 128Mi - # VMM reserve). The golden snapshot and cold boot stay small: the working - # set exists from the first fill onward. --mem-churn re-randomizes 64Mi + # target + headroom (glutton itself and the guest). The golden snapshot + # and cold boot stay small: the working set exists from the first fill + # onward. --mem-churn re-randomizes 64Mi # each cycle through an advancing window (WriteRAM rotate), so repeated # suspends snapshot changing memory rather than a static set and the dirty # window moves across the array. --mem-read walks the whole working set diff --git a/benchmarking/deploy_locust.sh b/benchmarking/deploy_locust.sh index fb4d335bdb..6d216125cd 100755 --- a/benchmarking/deploy_locust.sh +++ b/benchmarking/deploy_locust.sh @@ -29,7 +29,7 @@ WORKER_COUNT=1 SANDBOX_CLASS=gvisor SKIP_BUILD=0 OTLP_ENDPOINT="" -# Empty keeps the default in workloads/deploy.sh (256Mi, the microvm minimum). +# Empty keeps the default in workloads/deploy.sh (256Mi). ACTOR_MEMORY="" # Empty keeps the default in workloads/deploy.sh (1 CPU). ACTOR_CPU="" @@ -50,7 +50,7 @@ usage() { echo " --otlp-endpoint URL Forwarded to workloads/deploy.sh. The address to which an" echo " instrumented actor container sends telemetry." echo " --actor-memory SIZE Forwarded to workloads/deploy.sh. Memory limit for the" - echo " benchmark ActorTemplates (default: 256Mi, the microvm minimum)." + echo " benchmark ActorTemplates (default: 256Mi)." echo " --actor-cpu CPU Forwarded to workloads/deploy.sh. CPU limit for the benchmark" echo " ActorTemplates (default: 1)." echo " --worker-memory SIZE Forwarded to workloads/deploy.sh. Memory request and limit" diff --git a/benchmarking/workloads/deploy.sh b/benchmarking/workloads/deploy.sh index c18328dfca..010a2cc295 100755 --- a/benchmarking/workloads/deploy.sh +++ b/benchmarking/workloads/deploy.sh @@ -57,18 +57,17 @@ fi WORKER_COUNT=1 SANDBOX_CLASS="gvisor" -# Actor memory limit (ActorTemplate resources.limits.memory). The default -# is the smallest size microvm admits (128Mi VMM reserve + 128Mi guest floor), -# since a bigger guest drags more page cache into every memory snapshot. Raise -# it for RAM-consuming suites. +# Actor memory limit (ActorTemplate resources.limits.memory). Small by +# default, since a bigger guest drags more page cache into every memory +# snapshot. Raise it for RAM-consuming suites. ACTOR_MEMORY="256Mi" # Actor CPU limit (ActorTemplate resources.limits.cpu). The scheduler books # it against the worker's CPU capacity, so it bounds how many actors a worker # hosts at once as much as the memory limit does. ACTOR_CPU="1" # Worker pod memory (WorkerPool spec.template.resources), set as both request -# and limit. Empty leaves the pod unsized. The limit is also the memory the -# worker reports as its actor capacity. +# and limit. Empty leaves the pod unsized. The limit, less what the worker +# keeps for itself, is the memory it reports as its actor capacity. WORKER_MEMORY="" # The StorageClass for the glutton-storage external volume. Empty means the # template is not deployed. @@ -96,8 +95,7 @@ usage() { echo " --worker-count N Number of WorkerPool replicas (default: 1)" echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)." echo " microvm requires hack/install-microvm-deps.sh --install to have run." - echo " --actor-memory SIZE Memory limit for the benchmark ActorTemplates (default: 256Mi," - echo " the smallest size microvm admits)" + echo " --actor-memory SIZE Memory limit for the benchmark ActorTemplates (default: 256Mi)" echo " --actor-cpu CPU CPU limit for the benchmark ActorTemplates (default: 1)" echo " --worker-memory SIZE Memory request and limit for each WorkerPool pod" echo " (default: unset, the pod is unsized)" diff --git a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl index 5f9ced3242..57c8c5fbef 100644 --- a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl @@ -26,8 +26,7 @@ containers: command: ["/bin/sleep", "1000000"] # Keep the memory small: anything the micro-VM guest kernel caches rides # along in the memory snapshot, so a bigger guest means slower -# suspend/resume. microvm requires at least 256Mi (128Mi VMM reserve + -# 128Mi guest floor). +# suspend/resume. microvm requires at least 128Mi. resources: limits: cpuMillicores: "${ACTOR_CPU_MILLICORES}" diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index 26cca003a0..ab12828f3f 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -7682,6 +7682,83 @@ func Validate_RevertActorRequest( return errs } +// Validate_RuntimeOverhead validates an instance of RuntimeOverhead according +// to declarative validation rules in the API schema. +func Validate_RuntimeOverhead( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *v1.RuntimeOverhead) (errs field.ErrorList) { + + { // field v1.RuntimeOverhead.MemoryBytes + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 1099511627776); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *v1.RuntimeOverhead) *int64 { + return &oldObj.MemoryBytes + }) + errs = append(errs, fn(fldPath.Child("memory_bytes"), &obj.MemoryBytes, oldVal, oldObj != nil)...) + } + + { // field v1.RuntimeOverhead.MemoryBytesPerGib + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 1073741824); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *v1.RuntimeOverhead) *int64 { + return &oldObj.MemoryBytesPerGib + }) + errs = append(errs, fn(fldPath.Child("memory_bytes_per_gib"), &obj.MemoryBytesPerGib, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_SecurityContext validates an instance of SecurityContext according // to declarative validation rules in the API schema. func Validate_SecurityContext( @@ -11091,6 +11168,36 @@ func Validate_WorkerSandboxRuntime( errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) } + { // field v1.WorkerSandboxRuntime.ActorOverhead + fn := func( + fldPath *field.Path, + obj, oldObj *v1.RuntimeOverhead, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_RuntimeOverhead(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *v1.WorkerSandboxRuntime) *v1.RuntimeOverhead { + return oldObj.ActorOverhead + }) + errs = append(errs, fn(fldPath.Child("actor_overhead"), obj.ActorOverhead, oldVal, oldObj != nil)...) + } + return errs } diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 1f324ea860..5ae65943ce 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -407,12 +407,6 @@ func (w *ActorWorkflow) validateAssignedWorker(ctx context.Context, actorRef res return worker, nil } -// admittedResources is what an assignment books against its worker, or nil -// when the actor declared no limits and so reserves nothing. -func admittedResources(constraints scheduling.Constraints) *ateapi.Resources { - return constraints.Limits -} - // workerHoldingStaleClaim recovers a claim written before the Actor update. // It releases the claim if the Worker is no longer eligible. func (w *ActorWorkflow) workerHoldingStaleClaim(ctx context.Context, actor *ateapi.Actor, constraints scheduling.Constraints, relaxable bool) (*ateapi.Worker, error) { @@ -474,7 +468,7 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour }() constraints := schedulingConstraints(actor, actorTemplate) - snapRuntime := constraints.RequiredRuntime + snapRuntime := constraints.SnapshotRuntime class = constraints.SandboxClass relaxable := canRelaxRuntime(actor, constraints) @@ -514,9 +508,15 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour Name: actor.GetMetadata().GetName(), }, ActorUid: actor.GetMetadata().GetUid(), - // Record what this claim reserves so release returns the same amount. - Resources: admittedResources(constraints), } + // Record what this claim books so release returns the same amount. It + // includes the overhead the cached Worker reports, which admit checks + // again under the row lock. + booking, ok := scheduling.Booking(assignedWorker, constraints) + if !ok { + return nil, nil, errWorkerFilledUp + } + assignment.Resources = booking // The candidate came from a watch-fed cache, so it may already be full or no // longer eligible. The store re-asks under the Worker's row lock, where the @@ -528,6 +528,11 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour if !w.scheduler.Applies(fresh, constraints) || !w.scheduler.HasRoom(fresh, constraints) { return errWorkerFilledUp } + // The Worker re-registered with a different overhead since the cache + // saw it. Rare, and the retry reads the new one. + if got, _ := scheduling.Booking(fresh, constraints); !proto.Equal(got, booking) { + return errWorkerFilledUp + } bound = fresh return nil } @@ -537,8 +542,9 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour return nil, nil, fmt.Errorf("selected worker disappeared before claim: %w", store.ErrVersionConflict) } // errWorkerFilledUp leaves the cache alone: it reports no room for - // this actor's size, so forgetting the worker would hide it from - // every smaller actor until the next watch event or relist. + // this actor's size, or an overhead the cache hasn't seen yet, so + // forgetting the worker would hide it from every smaller actor until + // the next watch event or relist. return nil, nil, err } @@ -623,9 +629,11 @@ func schedulingConstraints(actor *ateapi.Actor, tmpl *ateapi.ActorTemplate) sche if node := actor.GetStatus().GetAssignedNode(); node != "" { requiredNodes = []string{node} } + required := hardware.NewRequirement(requiredSnapshotRuntime(snap, tmpl)) c := scheduling.Constraints{ SandboxClass: snap.GetSandboxRuntime().GetSandboxClass(), - RequiredRuntime: hardware.NewRequirement(requiredSnapshotRuntime(snap, tmpl)), + RequiredRuntime: required, + SnapshotRuntime: required, ActorSelector: labels.SelectorFromSet(labels.Set(actor.GetWorkerSelector().GetMatchLabels())), RequiredNodes: requiredNodes, Limits: tmpl.GetResources().GetLimits(), diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 0da9100150..894e632b9e 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -3578,3 +3578,27 @@ func TestAssignWorkerAttempt_WorkerWithoutNodeRejectsVolumes(t *testing.T) { }) } } + +// A Worker that re-registered with a different overhead since the cache saw +// it is refused under its row lock, so what the assignment records is what +// admit checked. +func TestAssignWorkerAttempt_RefusesChangedOverhead(t *testing.T) { + cached := contendedWorker("pod-a") + cached.Status.SandboxClasses[0].DefaultRuntime.ActorOverhead = &ateapi.RuntimeOverhead{MemoryBytes: 4 << 20} + sched := &filledUpScheduler{workers: []*ateapi.Worker{cached}} + w, actor, _ := newContendedWorkflow(t, sched) + locked := proto.Clone(cached).(*ateapi.Worker) + locked.Status.SandboxClasses[0].DefaultRuntime.ActorOverhead = &ateapi.RuntimeOverhead{MemoryBytes: 16 << 20} + w.store.(*contentionStore).workers[locked.GetMetadata().GetName()] = locked + + tmpl := &ateapi.ActorTemplate{Resources: &ateapi.ResourceRequirements{Limits: &ateapi.Resources{MemoryBytes: 256 << 20}}} + _, _, err := w.assignWorkerAttempt(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, tmpl) + if !errors.Is(err, errWorkerFilledUp) { + t.Fatalf("assignWorkerAttempt() = %v, want %v", err, errWorkerFilledUp) + } + + w.store.(*contentionStore).workers[locked.GetMetadata().GetName()] = cached + if _, _, err := w.assignWorkerAttempt(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, tmpl); err != nil { + t.Fatalf("assignWorkerAttempt() with an unchanged overhead = %v", err) + } +} diff --git a/cmd/ateapi/internal/scheduling/scheduling.go b/cmd/ateapi/internal/scheduling/scheduling.go index d43dd5aef5..0e6acd13c6 100644 --- a/cmd/ateapi/internal/scheduling/scheduling.go +++ b/cmd/ateapi/internal/scheduling/scheduling.go @@ -19,6 +19,8 @@ import ( "context" "errors" "fmt" + "math" + "math/bits" "math/rand" "slices" @@ -44,6 +46,12 @@ type Constraints struct { // asks it of every candidate. RequiredRuntime hardware.Requirement + // SnapshotRuntime is the runtime of the snapshot the actor restores from, + // or the zero Requirement when it starts on a worker's default runtime. + // Unlike RequiredRuntime it is never relaxed: it picks the worker runtime + // the actor runs under, and so the overhead it is booked with. + SnapshotRuntime hardware.Requirement + // TemplateSelector and ActorSelector must both match the worker's labels. TemplateSelector labels.Selector ActorSelector labels.Selector @@ -123,7 +131,11 @@ func (s *scheduler) Schedule(ctx context.Context, constraints Constraints) (*ate if !s.Applies(worker, constraints) { continue } - if cand, ok := checkRoom(worker, constraints.Limits); ok { + booking, ok := Booking(worker, constraints) + if !ok { + continue + } + if cand, ok := checkRoom(worker, booking); ok { candidates = append(candidates, cand) } } @@ -252,10 +264,60 @@ func (s *scheduler) Applies(worker *ateapi.Worker, constraints Constraints) bool // this size. A dimension the worker does not report is unconstrained, so // placement is never blocked by missing data. func (s *scheduler) HasRoom(worker *ateapi.Worker, constraints Constraints) bool { - _, ok := checkRoom(worker, constraints.Limits) + booking, ok := Booking(worker, constraints) + if !ok { + return false + } + _, ok = checkRoom(worker, booking) return ok } +// Booking is what an actor takes from the worker's capacity: its limits plus +// the overhead of the worker runtime it would run under there. It reports +// false when that doesn't fit in an int64, and the actor doesn't fit on the worker. +func Booking(worker *ateapi.Worker, constraints Constraints) (*ateapi.Resources, bool) { + overhead := constraints.SnapshotRuntime.WorkerRuntimeFor(worker.GetStatus()).GetActorOverhead() + if overhead == nil { + return constraints.Limits, true + } + limit := constraints.Limits.GetMemoryBytes() + scaled, ok := perGiB(limit, overhead.GetMemoryBytesPerGib()) + if !ok { + return nil, false + } + memory, ok := sum(limit, overhead.GetMemoryBytes(), scaled) + if !ok { + return nil, false + } + return &ateapi.Resources{CpuMillicores: constraints.Limits.GetCpuMillicores(), MemoryBytes: memory}, true +} + +// perGiB scales rate bytes per GiB to n bytes, and reports false on overflow. +func perGiB(n, rate int64) (int64, bool) { + if n < 0 || rate < 0 { + return 0, false + } + hi, lo := bits.Mul64(uint64(n), uint64(rate)) + // divide the 128-bit product by 1<<30 + if hi>>30 != 0 { + return 0, false + } + v := hi<<34 | lo>>30 + return int64(v), v <= math.MaxInt64 +} + +// sum adds non-negative values, and reports false on overflow. +func sum(vs ...int64) (int64, bool) { + var total int64 + for _, v := range vs { + if v < 0 || v > math.MaxInt64-total { + return 0, false + } + total += v + } + return total, true +} + func checkRoom(worker *ateapi.Worker, want *ateapi.Resources) (candidate, bool) { capacity := worker.GetStatus().GetCapacity() used := worker.GetStatus().GetAllocated() diff --git a/cmd/ateapi/internal/scheduling/scheduling_test.go b/cmd/ateapi/internal/scheduling/scheduling_test.go index 8e2985d52c..b06a1b0c21 100644 --- a/cmd/ateapi/internal/scheduling/scheduling_test.go +++ b/cmd/ateapi/internal/scheduling/scheduling_test.go @@ -17,11 +17,13 @@ package scheduling import ( "context" "errors" + "math" "testing" "go.ate.dev/substrate/api/ateapi/v1" "go.ate.dev/substrate/internal/hardware" "go.ate.dev/substrate/internal/resources" + "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/labels" ) @@ -712,3 +714,87 @@ func TestSchedulePowerOfTwoChoicesShiftedSecondSample(t *testing.T) { t.Fatalf("Schedule() = %q, want %q", got.GetWorkerPod(), "w-2") } } + +// withOverhead sets the actor overhead on the worker's runtimes: def on the +// default, restorable on the restorable ones in order. +func withOverhead(def *ateapi.RuntimeOverhead, restorable ...*ateapi.RuntimeOverhead) func(*ateapi.Worker) { + return func(w *ateapi.Worker) { + c := w.Status.SandboxClasses[0] + c.DefaultRuntime.ActorOverhead = def + for i, o := range restorable { + c.RestorableRuntimes[i].ActorOverhead = o + } + } +} + +func TestBooking(t *testing.T) { + const gib = 1 << 30 + limits := &ateapi.Resources{CpuMillicores: 500, MemoryBytes: gib / 2} + current, old := sandboxRuntime("microvm", "amd64", "new"), sandboxRuntime("microvm", "amd64", "old") + w := worker("w", "", "node-a", nil, + withRuntimes(current, old), + withOverhead(&ateapi.RuntimeOverhead{MemoryBytes: 4 << 20, MemoryBytesPerGib: 8 << 20}, &ateapi.RuntimeOverhead{MemoryBytes: 16 << 20})) + for _, tc := range []struct { + name string + constraints Constraints + want int64 + }{ + {name: "cold boot books the default runtime's", constraints: Constraints{Limits: limits}, want: gib/2 + 4<<20 + 4<<20}, + { + name: "a restore books the matching runtime's", + constraints: Constraints{Limits: limits, RequiredRuntime: hardware.NewRequirement(old), SnapshotRuntime: hardware.NewRequirement(old)}, + want: gib/2 + 16<<20, + }, + { + // relaxing what the worker must match doesn't change the runtime the + // actor restores under, so it doesn't change the price + name: "a relaxed restore still books the matching runtime's", + constraints: Constraints{Limits: limits, SnapshotRuntime: hardware.NewRequirement(old)}, + want: gib/2 + 16<<20, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, ok := Booking(w, tc.constraints) + if !ok || got.GetMemoryBytes() != tc.want || got.GetCpuMillicores() != 500 { + t.Errorf("Booking() = %v, %v, want %d memory bytes and 500 millicores", got, ok, tc.want) + } + }) + } + plain := worker("plain", "gvisor", "node-a", nil) + if got, ok := Booking(plain, Constraints{Limits: limits}); !ok || !proto.Equal(got, limits) { + t.Errorf("Booking() without overhead = %v, %v, want the limits %v", got, ok, limits) + } +} + +// A booking that overflows fits nowhere, rather than wrapping negative and +// fitting everywhere. +func TestBookingOverflow(t *testing.T) { + w := worker("w", "microvm", "node-a", nil, withMaxActors(10), withCapacity(0, 1<<30), + withOverhead(&ateapi.RuntimeOverhead{MemoryBytes: 4 << 20, MemoryBytesPerGib: 8 << 20})) + for _, memory := range []int64{math.MaxInt64, math.MaxInt64 - 1<<20} { + constraints := Constraints{Limits: &ateapi.Resources{MemoryBytes: memory}} + if got, ok := Booking(w, constraints); ok { + t.Errorf("Booking(%d) = %v, want overflow", memory, got) + } + if New(fleet{w}).HasRoom(w, constraints) { + t.Errorf("HasRoom(%d) = true, want false", memory) + } + } +} + +// An actor that fits the worker's free memory alone can still not fit with its +// runtime's overhead. +func TestHasRoomBooksRuntimeOverhead(t *testing.T) { + const gib = 1 << 30 + constraints := Constraints{Limits: &ateapi.Resources{MemoryBytes: gib}} + overhead := withOverhead(&ateapi.RuntimeOverhead{MemoryBytes: 4 << 20}) + tight := worker("tight", "microvm", "node-a", nil, withMaxActors(10), withCapacity(0, gib), overhead) + roomy := worker("roomy", "microvm", "node-a", nil, withMaxActors(10), withCapacity(0, gib+4<<20), overhead) + s := New(fleet{tight, roomy}) + if s.HasRoom(tight, constraints) { + t.Error("HasRoom() = true without room for the actor's overhead, want false") + } + if !s.HasRoom(roomy, constraints) { + t.Error("HasRoom() = false with room for the actor and its overhead, want true") + } +} diff --git a/cmd/atelet/internal/apivalidation/zz_generated.validation.go b/cmd/atelet/internal/apivalidation/zz_generated.validation.go index a312fc66a6..8b6884598a 100644 --- a/cmd/atelet/internal/apivalidation/zz_generated.validation.go +++ b/cmd/atelet/internal/apivalidation/zz_generated.validation.go @@ -3152,6 +3152,83 @@ func Validate_RunRequest( return errs } +// Validate_RuntimeOverhead validates an instance of RuntimeOverhead according +// to declarative validation rules in the API schema. +func Validate_RuntimeOverhead( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateletpb.RuntimeOverhead) (errs field.ErrorList) { + + { // field ateletpb.RuntimeOverhead.MemoryBytes + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 1099511627776); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.RuntimeOverhead) *int64 { + return &oldObj.MemoryBytes + }) + errs = append(errs, fn(fldPath.Child("memory_bytes"), &obj.MemoryBytes, oldVal, oldObj != nil)...) + } + + { // field ateletpb.RuntimeOverhead.MemoryBytesPerGib + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 1073741824); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.RuntimeOverhead) *int64 { + return &oldObj.MemoryBytesPerGib + }) + errs = append(errs, fn(fldPath.Child("memory_bytes_per_gib"), &obj.MemoryBytesPerGib, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_SecurityContext validates an instance of SecurityContext according // to declarative validation rules in the API schema. func Validate_SecurityContext( @@ -4624,6 +4701,36 @@ func Validate_WorkerSandboxRuntime( errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) } + { // field ateletpb.WorkerSandboxRuntime.ActorOverhead + fn := func( + fldPath *field.Path, + obj, oldObj *ateletpb.RuntimeOverhead, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_RuntimeOverhead(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.WorkerSandboxRuntime) *ateletpb.RuntimeOverhead { + return oldObj.ActorOverhead + }) + errs = append(errs, fn(fldPath.Child("actor_overhead"), obj.ActorOverhead, oldVal, oldObj != nil)...) + } + return errs } diff --git a/cmd/atelet/workersupport.go b/cmd/atelet/workersupport.go index 44fd8dab52..60ca5f6b8d 100644 --- a/cmd/atelet/workersupport.go +++ b/cmd/atelet/workersupport.go @@ -221,7 +221,19 @@ func toAteAPISandboxRuntime(in *ateletpb.WorkerSandboxRuntime) *ateapi.WorkerSan if in == nil { return nil } - return &ateapi.WorkerSandboxRuntime{Version: toAteAPICompat(in.GetVersion())} + return &ateapi.WorkerSandboxRuntime{ + Version: toAteAPICompat(in.GetVersion()), + ActorOverhead: toAteAPIOverhead(in.GetActorOverhead()), + } +} + +// toAteAPIOverhead converts atelet's RuntimeOverhead to the control plane's, +// which it mirrors field for field. +func toAteAPIOverhead(in *ateletpb.RuntimeOverhead) *ateapi.RuntimeOverhead { + if in == nil { + return nil + } + return &ateapi.RuntimeOverhead{MemoryBytes: in.GetMemoryBytes(), MemoryBytesPerGib: in.GetMemoryBytesPerGib()} } func toAteAPISandboxRuntimes(in []*ateletpb.WorkerSandboxRuntime) []*ateapi.WorkerSandboxRuntime { diff --git a/cmd/atelet/workersupport_test.go b/cmd/atelet/workersupport_test.go index 73d2c42972..8e596fddc1 100644 --- a/cmd/atelet/workersupport_test.go +++ b/cmd/atelet/workersupport_test.go @@ -325,6 +325,21 @@ func TestSetWorkerSandboxRuntimes(t *testing.T) { name: "records what the worker says", req: &ateletpb.SetWorkerSandboxRuntimesRequest{Epoch: 3, SandboxClasses: reqClasses()}, wantForwarded: forwarded(), + }, { + name: "forwards the runtimes' actor overhead", + req: func() *ateletpb.SetWorkerSandboxRuntimesRequest { + classes := reqClasses(reqCompat("d547")) + classes[0].DefaultRuntime.ActorOverhead = &ateletpb.RuntimeOverhead{MemoryBytes: 4 << 20, MemoryBytesPerGib: 8 << 20} + classes[0].RestorableRuntimes[0].ActorOverhead = &ateletpb.RuntimeOverhead{MemoryBytes: 16 << 20} + return &ateletpb.SetWorkerSandboxRuntimesRequest{Epoch: 3, SandboxClasses: classes} + }(), + wantForwarded: func() []*ateapi.SetWorkerSandboxRuntimesRequest { + want := forwarded(wantCompat("d547")) + c := want[0].GetSandboxClasses()[0] + c.DefaultRuntime.ActorOverhead = &ateapi.RuntimeOverhead{MemoryBytes: 4 << 20, MemoryBytesPerGib: 8 << 20} + c.RestorableRuntimes[0].ActorOverhead = &ateapi.RuntimeOverhead{MemoryBytes: 16 << 20} + return want + }(), }, { name: "forwards restorable runtimes", req: &ateletpb.SetWorkerSandboxRuntimesRequest{Epoch: 3, SandboxClasses: reqClasses(reqCompat("d547"))}, diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index 14f386a144..8cde90bf9c 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -92,6 +92,13 @@ var ( "Unix socket of atelet's OTLP relay to export telemetry through, keeping it off the pod network. Empty, or absent at startup, exports directly to OTEL_EXPORTER_OTLP_ENDPOINT instead.") ) +// memoryReservedBytes is how much of the worker pod's memory limit we keep from +// actors: the worker process itself, and the page cache of checkpoints in +// flight, which leaves the actor's own cgroup with its sentry (see +// isolateCheckpointCache). Otherwise an actor's sentry and gofers are charged +// to its own cgroup, so gVisor reports no actor overhead. +const memoryReservedBytes = 64 << 20 + // minUsageSampleInterval is the floor of --usage-sample-interval, the same as // atelet's poll interval floor. const minUsageSampleInterval = 50 * time.Second @@ -293,6 +300,7 @@ func do(ctx context.Context) error { AteletSPIFFEID: tunnelConfig.BrokerIdentity, Actors: *maxActors, PodIPs: register.ParsePodIPs(*podIPs), + MemoryReservedBytes: memoryReservedBytes, SandboxClass: string(atev1.SandboxClassGvisor), RuntimesFile: nodepath.WorkerSandboxRuntimesPath(*podUID), } diff --git a/cmd/ateworker-microvm/main.go b/cmd/ateworker-microvm/main.go index 9f8aac1783..b93b575031 100644 --- a/cmd/ateworker-microvm/main.go +++ b/cmd/ateworker-microvm/main.go @@ -64,14 +64,13 @@ import ( ) var ( - podUID = pflag.String("pod-uid", "", "The UID of the current pod") - podIPs = pflag.String("pod-ips", "", "The comma-separated IPs of the current pod") - chBinary = pflag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).") - guestDebug = pflag.Bool("guest-debug", false, "enable guest debugging (agent debug logs, debug console, and early serial console)") - vmmMemReserve = pflag.Int("vmm-mem-reserve-mib", vmmMemReserveMiB, "Guest RAM (MiB) held back from the pod's memory limit for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the pod cgroup alongside the guest RAM. Prevents the pod OOMing when the VM is sized to the pod's memory limit.") - showVersion = pflag.Bool("version", false, "Print version and exit.") - logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") - previewFlags = pflag.StringSlice("preview", nil, "Preview gates to enable.") + podUID = pflag.String("pod-uid", "", "The UID of the current pod") + podIPs = pflag.String("pod-ips", "", "The comma-separated IPs of the current pod") + chBinary = pflag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).") + guestDebug = pflag.Bool("guest-debug", false, "enable guest debugging (agent debug logs, debug console, and early serial console)") + showVersion = pflag.Bool("version", false, "Print version and exit.") + logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") + previewFlags = pflag.StringSlice("preview", nil, "Preview gates to enable.") otlpRelaySocket = pflag.String("otlp-relay-socket", nodepath.AteletOTLPSocketPath(), "Unix socket of atelet's OTLP relay to export telemetry through, keeping it off the pod network. Empty, or absent at startup, exports directly to OTEL_EXPORTER_OTLP_ENDPOINT instead.") @@ -233,7 +232,7 @@ func do(ctx context.Context) error { if err != nil { return err } - ateworkerService := NewService(*podUID, *chBinary, *guestDebug, *vmmMemReserve, *maxActors, tunnel, actorLogger) + ateworkerService := NewService(*podUID, *chBinary, *guestDebug, *maxActors, tunnel, actorLogger) // The controller sets both from the downward API. pool := workerstats.Pool{Namespace: os.Getenv("POD_NAMESPACE"), Name: os.Getenv("WORKER_POOL_NAME")} if pool.Namespace == "" || pool.Name == "" { @@ -286,6 +285,8 @@ func do(ctx context.Context) error { AteletSPIFFEID: tunnelConfig.BrokerIdentity, Actors: *maxActors, PodIPs: register.ParsePodIPs(*podIPs), + MemoryReservedBytes: memoryReservedBytes, + ActorOverhead: actorOverhead, SandboxClass: string(atev1.SandboxClassMicroVM), RuntimesFile: nodepath.WorkerSandboxRuntimesPath(*podUID), } @@ -372,11 +373,6 @@ type AteWorkerService struct { chBinary string guestDebug bool - // memReserveMiB is guest RAM (MiB) held back from the pod's memory limit for - // the cloud-hypervisor VMM + virtiofsd (host processes sharing the pod cgroup - // with the guest RAM). Set from --vmm-mem-reserve-mib. - memReserveMiB int - // actorLogger forwards the actor container's stdout/stderr to the worker pod's // stdout as ate.dev/*-labeled JSON and emits actor lifecycle events (parity // with ateworker-gvisor). @@ -400,19 +396,18 @@ type AteWorkerService struct { var _ ateworkerpb.WorkerServer = (*AteWorkerService)(nil) // NewService creates a new AteWorkerService. -func NewService(podUID, chBinary string, guestDebug bool, memReserveMiB, maxActors int, tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger) *AteWorkerService { +func NewService(podUID, chBinary string, guestDebug bool, maxActors int, tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger) *AteWorkerService { return &AteWorkerService{ - locks: actorlock.New(), - inFlight: actorlock.NewInFlight(), - actors: map[string]*hostedActor{}, - guestSlots: make(chan struct{}, statsFanOut), - maxActors: maxActors, - podUID: podUID, - chBinary: chBinary, - guestDebug: guestDebug, - memReserveMiB: memReserveMiB, - tunnel: tunnel, - actorLogger: actorLogger, + locks: actorlock.New(), + inFlight: actorlock.NewInFlight(), + actors: map[string]*hostedActor{}, + guestSlots: make(chan struct{}, statsFanOut), + maxActors: maxActors, + podUID: podUID, + chBinary: chBinary, + guestDebug: guestDebug, + tunnel: tunnel, + actorLogger: actorLogger, } } diff --git a/cmd/ateworker-microvm/run.go b/cmd/ateworker-microvm/run.go index 1b2fda9fc3..ffc403e709 100644 --- a/cmd/ateworker-microvm/run.go +++ b/cmd/ateworker-microvm/run.go @@ -42,6 +42,7 @@ import ( "go.ate.dev/substrate/internal/ateworkervalidation" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/ocispec" + "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/internal/sizing" @@ -135,30 +136,24 @@ const ( // boots it as PID 1 (init=), so it is the guest's entire userspace. const kataAgentPath = "/usr/bin/kata-agent" -// vmmMemReserveMiB is the DEFAULT guest RAM held back from the pod's memory limit -// for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the same -// pod cgroup as the guest RAM; without a margin the pod OOMs. Overridable per -// deployment via --vmm-mem-reserve-mib (see AteWorkerService.memReserveMiB). +// actorOverhead is what an actor costs outside its guest, in the same worker +// pod cgroup, which the control plane books with every actor: cloud-hypervisor +// and virtiofsd, and the host page tables for guest RAM, KVM's EPT and the +// memfd's xarray. // -// Measured on the worker pod's cgroup with one 256MiB-guest actor: the VMM stack's own -// cost is ~12MiB (anon 8.1 + kernel 3.7; the rest of cloud-hypervisor's RSS is the guest -// memfd, already accounted as guest RAM), and two virtiofsds are ~3MiB each. What needs -// the rest of the margin is transient: a pause/resume cycle took the cgroup from 94MiB -// to 153MiB, and the 57MiB difference was page cache from writing and reading the -// snapshot. -// -// That transient scales with snapshot size, so no fixed reserve is right for every guest -// size — this one is halved rather than cut to the ~32MiB the steady state would justify. -// The fix that would let it drop that far is keeping snapshot I/O out of the page cache -// (posix_fadvise(DONTNEED) after the checkpoint write and the restore read); until then -// the margin absorbs it, and a deployment running large guests can raise the flag. -const vmmMemReserveMiB = 128 - -// minGuestMemMiB is the floor for guest RAM (the declared limit minus the VMM -// reserve); a declared memory limit that leaves less is rejected at cold boot with a -// clear error instead of being silently honored (see resolveGuestMemMiB), since too -// little RAM makes the guest hang on boot rather than fail cleanly. Keep the admission -// floor on ActorTemplate.spec.resources in sync (it is this value + vmmMemReserveMiB). +// The fixed part also covers the page cache of the actor's snapshot writes, +// which are not capped yet, so tight pods don't OOM on suspend. +// TODO: drop that part once snapshot writes are capped +var actorOverhead = &ateletpb.RuntimeOverhead{MemoryBytes: 128 << 20, MemoryBytesPerGib: 8 << 20} + +// memoryReservedBytes is how much of the worker pod's memory limit we keep from +// actors: the worker process itself, and the page cache of snapshot writes in +// flight. +const memoryReservedBytes = 64 << 20 + +// minGuestMemMiB is the floor for guest RAM, and so for an actor's memory limit. A +// smaller one is rejected at cold boot (see resolveGuestMemMiB), since too little +// RAM makes the guest hang on boot rather than fail cleanly. // // Measured against the counter demo on a guest booting the agent as PID 1: 32MiB never // reaches Ready, 64MiB boots but idles with 1.1MB free (it only survives because page @@ -455,8 +450,8 @@ func (s *AteWorkerService) coldBootActor(ctx context.Context, p actorBootParams) kparams := s.kernelParams() // Size the VM to the actor's declared limits (see internal/sizing), which the - // API requires. vCPUs round up; VM RAM reserves a fixed margin for the VMM + - // virtiofsd, which share the pod cgroup with the guest RAM. NB: a MEMORY + // API requires. vCPUs round up; VM RAM is the memory limit, and what the actor + // costs outside the guest is booked as its overhead (see actorOverhead). NB: a MEMORY // snapshot restore reuses the size baked into the snapshot // (restoreMemoryFidelity), so resizing an existing actor takes effect on its // next cold boot. @@ -464,7 +459,7 @@ func (s *AteWorkerService) coldBootActor(ctx context.Context, p actorBootParams) if vcpus == 0 { return errors.New("actor has no cpu limit to size the micro-VM's vCPUs") } - memMiB, err := resolveGuestMemMiB(p.size.MemoryBytes, s.memReserveMiB) + memMiB, err := resolveGuestMemMiB(p.size.MemoryBytes) if err != nil { return err } @@ -481,10 +476,8 @@ func (s *AteWorkerService) coldBootActor(ctx context.Context, p actorBootParams) // template spec is immutable, so an actor's limits were validated when its // golden was built. if err := checkResourceEnvelope(ctrs, guestEnvelope{ - memMiB: memMiB, - vcpus: vcpus, - declaredBytes: p.size.MemoryBytes, - reserveMiB: s.memReserveMiB, + memMiB: memMiB, + vcpus: vcpus, }); err != nil { return err } @@ -747,19 +740,16 @@ func (s *AteWorkerService) kernelParams() string { } // resolveGuestMemMiB returns the micro-VM guest RAM (MiB) for an actor's declared -// memory limit: the declared memory minus the VMM reserve. It errors if the limit is -// unset or leaves less than a bootable minimum (see vmmMemReserveMiB, minGuestMemMiB, -// and internal/sizing). -func resolveGuestMemMiB(declaredBytes int64, reserveMiB int) (int, error) { +// memory limit, which is all of it (see actorOverhead). It errors if the limit +// is unset or below minGuestMemMiB. +func resolveGuestMemMiB(declaredBytes int64) (int, error) { if declaredBytes <= 0 { return 0, errors.New("actor has no memory limit to size the micro-VM's RAM") } - declaredMiB := int(declaredBytes / (1024 * 1024)) - m := declaredMiB - reserveMiB + m := int(declaredBytes / (1024 * 1024)) if m < minGuestMemMiB { - return 0, fmt.Errorf("actor memory limit %dMiB is too small for a micro-VM: "+ - "the %dMiB VMM reserve leaves %dMiB, below the %dMiB guest minimum", - declaredMiB, reserveMiB, m, minGuestMemMiB) + return 0, fmt.Errorf("actor memory limit %dMiB is too small for a microVM, the minimum is %dMiB", + m, minGuestMemMiB) } return m, nil } diff --git a/cmd/ateworker-microvm/run_test.go b/cmd/ateworker-microvm/run_test.go index ee6b71d55e..6e7c2bd666 100644 --- a/cmd/ateworker-microvm/run_test.go +++ b/cmd/ateworker-microvm/run_test.go @@ -103,13 +103,10 @@ func TestDialAgentRetryContextCanceled(t *testing.T) { } } -// resolveGuestMemMiB must honor a declared limit (minus the VMM reserve) and error -// when the limit is unset or the reserve leaves too little to boot a guest. +// resolveGuestMemMiB gives the guest the whole declared limit, and errors when +// the limit is unset or too small to boot a guest. func TestResolveGuestMemMiB(t *testing.T) { - const ( - mib = 1024 * 1024 - reserve = 256 // vmmMemReserveMiB - ) + const mib = 1024 * 1024 tests := []struct { name string declaredMiB int64 // 0 => unset @@ -117,15 +114,13 @@ func TestResolveGuestMemMiB(t *testing.T) { wantErr bool }{ {name: "unset", declaredMiB: 0, wantErr: true}, - {name: "declared honored minus reserve", declaredMiB: 1536, wantMiB: 1536 - reserve}, - {name: "just above minimum", declaredMiB: reserve + minGuestMemMiB, wantMiB: minGuestMemMiB}, - {name: "reserve exactly swallows limit", declaredMiB: reserve, wantErr: true}, - {name: "limit below reserve", declaredMiB: 128, wantErr: true}, - {name: "boot-hang band (too little guest RAM)", declaredMiB: 320, wantErr: true}, + {name: "declared honored", declaredMiB: 1536, wantMiB: 1536}, + {name: "minimum", declaredMiB: minGuestMemMiB, wantMiB: minGuestMemMiB}, + {name: "below minimum", declaredMiB: minGuestMemMiB - 1, wantErr: true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got, err := resolveGuestMemMiB(tc.declaredMiB*mib, reserve) + got, err := resolveGuestMemMiB(tc.declaredMiB * mib) if tc.wantErr { if err == nil { t.Fatalf("resolveGuestMemMiB(%dMiB) = %d, nil; want an error", tc.declaredMiB, got) diff --git a/cmd/ateworker-microvm/spec.go b/cmd/ateworker-microvm/spec.go index 21eec6af87..3342d95f8a 100644 --- a/cmd/ateworker-microvm/spec.go +++ b/cmd/ateworker-microvm/spec.go @@ -17,7 +17,6 @@ package main import ( - "fmt" "math" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -27,34 +26,24 @@ import ( ) // guestEnvelope is the ceiling an actor's container limits must fit inside. -// declaredBytes is the actor-level memory limit and reserveMiB the guest RAM -// held back for the VMM; together they explain where memMiB came from, so the -// error can name the field the user can actually raise. // -// memMiB is already net of reserveMiB, while vcpus is not. The asymmetry is -// deliberate: an unreduced memory limit would push the worker pod past its own -// and get it OOM-killed, whereas CPU is compressible, so the shortfall only -// slows the workload down. cloud-hypervisor's vCPU threads, virtiofsd and ateworker -// are host processes drawing on the same worker-pod CPU quota, and the -// scheduler's capacity check sets none of it aside, so a container gets somewhat -// less CPU than it declared and the in-guest quota is throttled by the host -// before it binds. Carving out a CPU reserve is left for a follow-up. +// memMiB is the actor's whole memory limit, and vcpus its CPU limit rounded up. +// cloud-hypervisor's vCPU threads, virtiofsd and ateworker are host processes +// drawing on the same worker-pod CPU quota, and the scheduler's capacity check +// sets none of it aside, so a container gets somewhat less CPU than it declared +// and the in-guest quota is throttled by the host before it binds. Carving out a +// CPU reserve is left for a follow-up. type guestEnvelope struct { - memMiB int - vcpus int - declaredBytes int64 - reserveMiB int + memMiB int + vcpus int } // remedy names the field that raises the memory ceiling. func (e guestEnvelope) remedy() string { - return fmt.Sprintf("raise spec.resources.limits.memory (declared %dMiB, less the %dMiB VMM reserve) or lower the container limits", - e.declaredBytes/(1024*1024), e.reserveMiB) + return "raise spec.resources.limits.memory or lower the container limits" } -// cpuRemedy names the field that raises the vCPU ceiling. Unlike remedy, it -// takes no reserve into account: the VMM reserve applies to guest memory, not -// vCPU count (see guestEnvelope). +// cpuRemedy names the field that raises the vCPU ceiling. func (e guestEnvelope) cpuRemedy() string { return "raise spec.resources.limits.cpu or lower the container limits" } diff --git a/cmd/ateworker-microvm/spec_test.go b/cmd/ateworker-microvm/spec_test.go index 05ec03908d..75efcaa5b0 100644 --- a/cmd/ateworker-microvm/spec_test.go +++ b/cmd/ateworker-microvm/spec_test.go @@ -101,7 +101,7 @@ func TestCheckResourceEnvelope(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - err := checkResourceEnvelope(tc.ctrs, guestEnvelope{memMiB: 2048, vcpus: 1, declaredBytes: (2048 + 256) * 1024 * 1024, reserveMiB: 256}) + err := checkResourceEnvelope(tc.ctrs, guestEnvelope{memMiB: 2048, vcpus: 1}) if tc.wantErr == "" { if err != nil { t.Fatalf("checkResourceEnvelope() = %v, want nil", err) @@ -121,25 +121,22 @@ func TestCheckResourceEnvelope(t *testing.T) { } } -// When the actor declared its own size, the guest ceiling comes from that limit -// minus the VMM reserve, so the error must point at the actor's limit rather -// than at the SandboxConfig the user cannot usefully change. +// The guest ceiling is the actor's own memory limit, so the error must point at +// that rather than at the SandboxConfig the user cannot usefully change. func TestCheckResourceEnvelope_ErrorNamesActorLimitWhenDeclared(t *testing.T) { const mib = 1024 * 1024 ctr := actorContainer{name: "hog", spec: &specs.Spec{Linux: &specs.Linux{ Resources: &specs.LinuxResources{Memory: &specs.LinuxMemory{Limit: ptr.To(int64(2048 * mib))}}, }}} - err := checkResourceEnvelope([]actorContainer{ctr}, guestEnvelope{ - memMiB: 768, vcpus: 1, declaredBytes: 1024 * mib, reserveMiB: 256, - }) + err := checkResourceEnvelope([]actorContainer{ctr}, guestEnvelope{memMiB: 1024, vcpus: 1}) if err == nil { t.Fatal("checkResourceEnvelope() = nil, want an error") } if got := apierror.Code(err); got != codes.InvalidArgument { t.Errorf("code = %v, want InvalidArgument", got) } - for _, want := range []string{"hog", "1024", "256", "spec.resources.limits.memory"} { + for _, want := range []string{"hog", "1024", "spec.resources.limits.memory"} { if !strings.Contains(err.Error(), want) { t.Errorf("error %q does not mention %q", err.Error(), want) } @@ -150,16 +147,13 @@ func TestCheckResourceEnvelope_ErrorNamesActorLimitWhenDeclared(t *testing.T) { // spec.resources.limits.memory, even when the actor also declares a memory // limit: raising memory cannot change the vCPU count. func TestCheckResourceEnvelope_CPUErrorNamesCPULimitEvenWithDeclaredMemory(t *testing.T) { - const mib = 1024 * 1024 period := uint64(kata.DefaultCPUPeriodUS) quota := int64(2000 * kata.DefaultCPUPeriodUS / 1000) // 2000m ctr := actorContainer{name: "hog", spec: &specs.Spec{Linux: &specs.Linux{ Resources: &specs.LinuxResources{CPU: &specs.LinuxCPU{Quota: "a, Period: &period}}, }}} - err := checkResourceEnvelope([]actorContainer{ctr}, guestEnvelope{ - memMiB: 2048, vcpus: 1, declaredBytes: 2048 * mib, reserveMiB: 256, - }) + err := checkResourceEnvelope([]actorContainer{ctr}, guestEnvelope{memMiB: 2048, vcpus: 1}) if err == nil { t.Fatal("checkResourceEnvelope() = nil, want an error") } @@ -174,30 +168,22 @@ func TestCheckResourceEnvelope_CPUErrorNamesCPULimitEvenWithDeclaredMemory(t *te } } -// The ceiling containers are measured against is the post-reserve guest, not the -// declared limit and not the SandboxConfig default. RunWorkload gets there by -// calling resolveGuestMemMiB before checkResourceEnvelope; this pins the -// arithmetic those two steps compose into, which nothing else asserts. -func TestCheckResourceEnvelope_MeasuresAgainstPostReserveGuest(t *testing.T) { +// A container may ask for the actor's whole memory limit, since the guest gets +// all of it. RunWorkload gets there by calling resolveGuestMemMiB before +// checkResourceEnvelope; this pins the arithmetic those two steps compose into. +func TestCheckResourceEnvelope_ContainerMayUseWholeLimit(t *testing.T) { const mib = 1024 * 1024 - const declaredMiB, reserveMiB = 1024, 256 + const declaredMiB = 1024 - guestMiB, err := resolveGuestMemMiB(int64(declaredMiB)*mib, reserveMiB) + guestMiB, err := resolveGuestMemMiB(int64(declaredMiB) * mib) if err != nil { t.Fatalf("resolveGuestMemMiB() = %v", err) } - if guestMiB != declaredMiB-reserveMiB { - t.Fatalf("guest = %dMiB, want %dMiB (declared minus reserve)", guestMiB, declaredMiB-reserveMiB) - } - - // A container asking for the full declared limit does not fit the guest, - // because the reserve is not the container's to spend. - ctr := actorContainer{name: "hog", spec: &specs.Spec{Linux: &specs.Linux{ + ctr := actorContainer{name: "whole", spec: &specs.Spec{Linux: &specs.Linux{ Resources: &specs.LinuxResources{Memory: &specs.LinuxMemory{Limit: ptr.To(int64(declaredMiB) * mib)}}, }}} - env := guestEnvelope{memMiB: guestMiB, vcpus: 1, declaredBytes: int64(declaredMiB) * mib, reserveMiB: reserveMiB} - if err := checkResourceEnvelope([]actorContainer{ctr}, env); err == nil { - t.Error("checkResourceEnvelope() = nil, want an error: the declared limit does not fit once the reserve is held back") + if err := checkResourceEnvelope([]actorContainer{ctr}, guestEnvelope{memMiB: guestMiB, vcpus: 1}); err != nil { + t.Errorf("checkResourceEnvelope() = %v, want nil", err) } } diff --git a/demos/counter/counter-microvm-template.yaml.tmpl b/demos/counter/counter-microvm-template.yaml.tmpl index 052bc209b6..9e883f6d82 100644 --- a/demos/counter/counter-microvm-template.yaml.tmpl +++ b/demos/counter/counter-microvm-template.yaml.tmpl @@ -50,9 +50,8 @@ containers: volumeMounts: - name: data mountPath: /home/counter -# Sandbox size: vCPUs + guest RAM at (or below) the pool's per-worker -# capacity, leaving VMM-reserve headroom under the worker memory limit -# (see internal/sizing). +# Sandbox size: vCPUs + guest RAM, which with the microVM overhead has to +# fit the pool's per-worker capacity (see docs/api-guide.md). resources: limits: cpuMillicores: 1000 diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index 56381dd579..d609850dbf 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -48,15 +48,16 @@ spec: # Per-worker resources: these size the worker POD and advertise its # scheduling capacity (the per-actor ceiling). The micro-VM sandbox itself is # sized by the ActorTemplate's resources in counter-microvm-template.yaml.tmpl, - # not by these; keep the actor memory below this limit so the VMM reserve - # fits (see internal/sizing). + # not by these. The worker keeps some of this limit for itself, and each + # actor books its limit plus the microVM overhead, so keep this limit + # above what the actors need together (see docs/api-guide.md). # # Capacity comes from the limits (workerCapacity reads limits only), while the # kube-scheduler packs nodes by the requests — so the CPU request is set well # below the limit to keep the pool placeable on a small node (a single kind # node in CI, shared with the gVisor demo pool) without changing what an actor # is allowed to use. Memory is not compressible, so its request matches the - # limit: the guest RAM plus the VMM reserve has to be really there. + # limit: the guest RAM plus its overhead has to be really there. resources: limits: cpu: "1" diff --git a/docs/api-guide.md b/docs/api-guide.md index e72afcdf19..ffb19a45ff 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -43,6 +43,7 @@ Setting `resources.limits` (CPU and Memory) on a `WorkerPool` establishes each w - Worker capacity is a shared budget: each actor placed on a worker subtracts its declared limits from what is left. Size a pool's `limits` for the actors it should host together. - A worker also has an actor limit, set by the ateworker's `--max-actors` flag (default 1000). Placement stops at whichever runs out first. +- A worker keeps some of its memory limit for itself and reports the rest as capacity. Each actor can also use memory on its worker outside its own limits, e.g. for the microVM's VMM. The worker reports this overhead with each runtime, placement counts it against capacity, and it shows in the worker's allocated resources. - Capacity is advisory for placement only: a worker that declares no CPU/memory limit reports zero capacity for that dimension, which the scheduler treats as **unconstrained** (placement is never blocked by missing data). The actual sandbox size still comes from the `ActorTemplate`. ### Example @@ -160,8 +161,8 @@ Unlike a Pod, an actor is sized by its **`limits`** (CPU and Memory): the size i 1. **Size the sandbox.** The limits are supplied to the sandbox over the actor RPCs (control plane → atelet → ateworker): - **gVisor (`ateworker-gvisor`)** — applied to the container OCI spec: `limits.cpuMillicores` sets the cgroup v2 CPU quota (`cpu.max`) and the Sentry vCPU count (`--cpu-num-from-quota`); `limits.memoryBytes` caps the sandbox's memory and bounds the virtual total memory it reports (so JVM/Go do not over-allocate from host RAM). When the node's kernel has cgroup memory pressure (PSI) enabled, an actor that stays over its limit is killed and reported crashed, and other actors on its worker keep running. - - **microVM (`ateworker-microvm`)** — `limits.cpuMillicores` sets Cloud Hypervisor `BootVcpus` / `MaxVcpus` (rounded up to whole vCPUs); `limits.memoryBytes` sets guest RAM, reserving a small configurable margin (default 256 MiB, `--vmm-mem-reserve-mib`) for the VMM and virtiofsd so the pod cgroup does not OOM. -2. **Gate scheduling.** An actor is only placed on a `WorkerPool` whose [worker capacity](#worker-capacity-spectemplateresources) is `>=` these limits. + - **microVM (`ateworker-microvm`)** — `limits.cpuMillicores` sets Cloud Hypervisor `BootVcpus` / `MaxVcpus` (rounded up to whole vCPUs); `limits.memoryBytes` sets guest RAM, all of it. The microVM worker refuses to boot an actor with less than 128 MiB. What the actor costs outside the guest comes out of the worker's capacity instead (see [Worker Capacity](#worker-capacity-spectemplateresources)). +2. **Gate scheduling.** An actor is only placed on a `WorkerPool` whose [worker capacity](#worker-capacity-spectemplateresources) is `>=` these limits, plus its runtime's overhead on that worker. `requests` are not consulted today (an actor occupies its whole worker). Because the size is baked into snapshots, a **microVM MEMORY-fidelity restore reuses the size in the snapshot**; changing an actor's limits takes effect on its next cold boot. @@ -326,7 +327,7 @@ A container that exceeds its memory limit is OOM-killed on its own; the actor's Per-container limits are microVM only today. gVisor applies cgroup limits at the sandbox level: one sentry backs every container in the actor, so a per-container cgroup is created and then stays empty ([google/gvisor#190](https://github.com/google/gvisor/issues/190)). On a gVisor worker the per-container limits are accepted but not enforced; only the sandbox-level size applies. -These limits subdivide the sandbox that [`resources`](#sandbox-right-sizing-resources) already sized; a container that declares none is bounded by the guest as a whole, not by a copy of the actor's total. A microVM guest's memory is `resources.limits.memoryBytes` minus the VMM reserve, and its vCPU count is `resources.limits.cpuMillicores` rounded up to whole cores. A container limit above either ceiling can never bind, so the actor fails to start with an error naming both the limit and the ceiling. +These limits subdivide the sandbox that [`resources`](#sandbox-right-sizing-resources) already sized; a container that declares none is bounded by the guest as a whole, not by a copy of the actor's total. A microVM guest's memory is `resources.limits.memoryBytes`, and its vCPU count is `resources.limits.cpuMillicores` rounded up to whole cores. A container limit above either ceiling can never bind, so the actor fails to start with an error naming both the limit and the ceiling. Each limit is validated on its own at apply, but the sum across the actor's containers is only checked when the actor first runs, against the real guest size. A template whose limits do not fit is accepted by the API server and fails on its first actor. diff --git a/internal/ateworker/register/register.go b/internal/ateworker/register/register.go index 36b6c03f10..ca523102f5 100644 --- a/internal/ateworker/register/register.go +++ b/internal/ateworker/register/register.go @@ -98,16 +98,26 @@ func (e *Epoch) Require() (int64, error) { // TODO: Watch the projected files and report changes. For now we do not support // in-place Pod vertical scaling (IPPR); capacity is read once at startup. // NOTE: Please do not implement this yet. IPPR needs more general consideration. -func FromFiles(actors int) *ateletpb.WorkerResources { - return fromDir(workercapacity.MountPath, actors) +// +// memoryReserved is how much of the memory limit the worker keeps for itself +// rather than offering to actors. +func FromFiles(actors int, memoryReserved int64) (*ateletpb.WorkerResources, error) { + return fromDir(workercapacity.MountPath, actors, memoryReserved) } -func fromDir(dir string, actors int) *ateletpb.WorkerResources { +func fromDir(dir string, actors int, memoryReserved int64) (*ateletpb.WorkerResources, error) { + memory := readLimit(filepath.Join(dir, workercapacity.MemoryLimitFile)) + if memory > 0 { + if memory <= memoryReserved { + return nil, fmt.Errorf("pod memory limit %d bytes is no more than the %d the worker keeps for itself, raise the pool's limits.memory", memory, memoryReserved) + } + memory -= memoryReserved + } return &ateletpb.WorkerResources{ CpuMillicores: readLimit(filepath.Join(dir, workercapacity.CPULimitFile)), - MemoryBytes: readLimit(filepath.Join(dir, workercapacity.MemoryLimitFile)), + MemoryBytes: memory, Actors: int32(actors), - } + }, nil } // ParsePodIPs splits the comma-separated pod IPs the downward API exposes for @@ -161,6 +171,20 @@ func buildRuntimesRequest(epoch int64, class *sandboxruntimes.Class) *ateletpb.S } } +// setActorOverhead reports a copy of overhead on every runtime in req. The +// restorable runtimes run under this worker too, so they cost it the same. +func setActorOverhead(req *ateletpb.SetWorkerSandboxRuntimesRequest, overhead *ateletpb.RuntimeOverhead) { + if overhead == nil { + return + } + for _, c := range req.GetSandboxClasses() { + c.GetDefaultRuntime().ActorOverhead = proto.CloneOf(overhead) + for _, rt := range c.GetRestorableRuntimes() { + rt.ActorOverhead = proto.CloneOf(overhead) + } + } +} + // classOf returns f's entry for class, or nil when the file has none. func classOf(f *sandboxruntimes.File, class string) *sandboxruntimes.Class { for i := range f.SandboxClasses { @@ -200,6 +224,12 @@ type ReportConfig struct { Actors int // PodIPs are the worker pod's IPs, from the downward API. PodIPs []string + // MemoryReservedBytes is how much of its memory limit the worker keeps for + // itself rather than reporting as capacity. + MemoryReservedBytes int64 + // ActorOverhead is what each actor costs the worker on top of its own + // limits, reported on every runtime. Nil is none. + ActorOverhead *ateletpb.RuntimeOverhead // SandboxClass is the class this worker implements ("gvisor", "microvm"). SandboxClass string // RuntimesFile is the sandbox runtimes file atelet keeps for this worker @@ -242,9 +272,13 @@ func Report(ctx context.Context, cfg ReportConfig, epoch *Epoch) error { if err != nil { return fmt.Errorf("register worker: %w", err) } + capacity, err := FromFiles(cfg.Actors, cfg.MemoryReservedBytes) + if err != nil { + return fmt.Errorf("register worker: %w", err) + } client := &dialingClient{socketPath: cfg.SocketPath, tlsConfig: tlsConfig} changes := WatchFilePath(ctx, cfg.RuntimesFile) - return run(ctx, cfg, FromFiles(cfg.Actors), client, epoch, changes) + return run(ctx, cfg, capacity, client, epoch, changes) } // run registers, then reports runtimes until ctx ends or the epoch is refused. @@ -414,6 +448,7 @@ func (r *reconciler) sync(ctx context.Context) (retry bool, err error) { return false, nil } req := buildRuntimesRequest(r.epoch, class) + setActorOverhead(req, r.cfg.ActorOverhead) if proto.Equal(req, r.reported) { return false, nil } diff --git a/internal/ateworker/register/register_test.go b/internal/ateworker/register/register_test.go index 97570e9e78..beefb454a6 100644 --- a/internal/ateworker/register/register_test.go +++ b/internal/ateworker/register/register_test.go @@ -62,7 +62,10 @@ func TestFromFiles(t *testing.T) { t.Fatalf("writing memory limit: %v", err) } - got := fromDir(dir, testActors) + got, err := fromDir(dir, testActors, 0) + if err != nil { + t.Fatal(err) + } if diff := cmp.Diff(tc.want, got, protocmp.Transform()); diff != "" { t.Errorf("reported capacity mismatch (-want +got):\n%s", diff) } @@ -71,13 +74,47 @@ func TestFromFiles(t *testing.T) { } func TestFromFilesMissing(t *testing.T) { - got := fromDir(t.TempDir(), testActors) + got, err := fromDir(t.TempDir(), testActors, 1<<20) + if err != nil { + t.Fatal(err) + } want := &ateletpb.WorkerResources{Actors: testActors} if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { t.Errorf("unset environment capacity mismatch (-want +got):\n%s", diff) } } +func TestFromFilesMemoryReserved(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, workercapacity.MemoryLimitFile), []byte("1073741824"), 0o600); err != nil { + t.Fatalf("writing memory limit: %v", err) + } + for _, tc := range []struct { + name string + reserved int64 + want int64 + wantErr bool + }{ + {name: "some", reserved: 64 << 20, want: 960 << 20}, + {name: "all of it", reserved: 1 << 30, wantErr: true}, + {name: "more than all of it", reserved: 2 << 30, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := fromDir(dir, testActors, tc.reserved) + if (err != nil) != tc.wantErr { + t.Fatalf("fromDir() error = %v, wantErr %v", err, tc.wantErr) + } + if tc.wantErr { + return + } + want := &ateletpb.WorkerResources{Actors: testActors, MemoryBytes: tc.want} + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("reported capacity mismatch (-want +got):\n%s", diff) + } + }) + } +} + func TestBuildCompat(t *testing.T) { got := buildCompat(sandboxruntimes.Attributes{ hardware.AttrSandboxBundleHash: "sha256:abc", @@ -845,3 +882,38 @@ func TestReportFailsFastOnMissingConfig(t *testing.T) { }) } } + +func TestReconcilerSyncReportsActorOverhead(t *testing.T) { + path := filepath.Join(t.TempDir(), nodepath.SandboxRuntimesFileName) + f := gvisorFile(sandboxruntimes.Attributes{hardware.AttrSandboxBundleHash: "sha256:v2"}, + sandboxruntimes.Attributes{hardware.AttrSandboxBundleHash: "sha256:v1"}) + if _, err := sandboxruntimes.Write(path, f); err != nil { + t.Fatal(err) + } + overhead := &ateletpb.RuntimeOverhead{MemoryBytes: 4 << 20, MemoryBytesPerGib: 8 << 20} + var got *ateletpb.SetWorkerSandboxRuntimesRequest + r := newReconciler( + ReportConfig{SandboxClass: "gvisor", RuntimesFile: path, ActorOverhead: overhead}, + 3, + func() (*sandboxruntimes.File, error) { return sandboxruntimes.Read(path) }, + func(_ context.Context, req *ateletpb.SetWorkerSandboxRuntimesRequest) error { + got = req + return nil + }, + ) + if retry, err := r.sync(context.Background()); retry || err != nil { + t.Fatalf("sync() = %v, %v, want false, nil", retry, err) + } + c := got.GetSandboxClasses()[0] + for _, rt := range append([]*ateletpb.WorkerSandboxRuntime{c.GetDefaultRuntime()}, c.GetRestorableRuntimes()...) { + if diff := cmp.Diff(overhead, rt.GetActorOverhead(), protocmp.Transform()); diff != "" { + t.Errorf("actor_overhead mismatch (-want +got):\n%s", diff) + } + if rt.GetActorOverhead() == overhead { + t.Error("runtime shares the worker's actor_overhead message") + } + } + if len(c.GetRestorableRuntimes()) != 1 { + t.Errorf("reported %d restorable runtimes, want 1", len(c.GetRestorableRuntimes())) + } +} diff --git a/internal/benchmarking/boomer/agentsession/scripts/personal-assistant-light.yaml b/internal/benchmarking/boomer/agentsession/scripts/personal-assistant-light.yaml index 7772e07bbb..60273a2f68 100644 --- a/internal/benchmarking/boomer/agentsession/scripts/personal-assistant-light.yaml +++ b/internal/benchmarking/boomer/agentsession/scripts/personal-assistant-light.yaml @@ -30,7 +30,7 @@ # an actor with a 512Mi memory limit and capped near 125 MB with a 256Mi # limit, plus a rootfs tar of up to 22 MB. # -# min_actor_memory is 256Mi: the script fits a 128 MiB micro-VM guest. +# min_actor_memory is 256Mi. # Run it like personal-assistant, with a fractional # --agentsession-think-scale. name: personal-assistant-light diff --git a/internal/e2e/suites/sizing/sizing_test.go b/internal/e2e/suites/sizing/sizing_test.go index b56e6f59f4..f853cd6d20 100644 --- a/internal/e2e/suites/sizing/sizing_test.go +++ b/internal/e2e/suites/sizing/sizing_test.go @@ -93,10 +93,9 @@ func TestActorSizing_SandboxObservesDeclaredLimits(t *testing.T) { } // Memory: the sandbox must be bounded by the declared limit. Both runtimes - // report under it — gVisor by its reserved overhead, the micro-VM by the - // 128MiB VMM reserve held back from the guest plus what the guest kernel - // takes — but neither may see more; a value near the node's full RAM means - // the limit was not applied. Allow 10% headroom above the limit for + // report under it — gVisor by its reserved overhead, the microVM by what + // the guest kernel takes — but neither may see more; a value near the + // node's full RAM means the limit was not applied. Allow 10% headroom above the limit for // accounting differences, and half the limit below it for those overheads. if got.MemTotalError != "" { t.Errorf("probe could not read MemTotal: %s", got.MemTotalError) diff --git a/internal/hardware/hardware.go b/internal/hardware/hardware.go index 6848b6a5c7..262846d651 100644 --- a/internal/hardware/hardware.go +++ b/internal/hardware/hardware.go @@ -189,37 +189,37 @@ func runtimeOf(class string, compat *ateapi.VersionedSandboxCompat) *ateapi.Acto return &ateapi.ActorSandboxRuntime{SandboxClass: class, Version: compat} } -// matchingCompat returns the version from status that can restore the +// matchingRuntime returns the runtime from status that can restore the // snapshot: in the worker's entry for the snapshot's class, DefaultRuntime // first, then RestorableRuntimes in order. nil if the worker does not run // that class or nothing in its entry matches. Only for a requirement of // something. -func (r Requirement) matchingCompat(status *ateapi.WorkerStatus) *ateapi.VersionedSandboxCompat { +func (r Requirement) matchingRuntime(status *ateapi.WorkerStatus) *ateapi.WorkerSandboxRuntime { c := classOf(status, r.snap.GetSandboxClass()) if c == nil { return nil } - if def := c.GetDefaultRuntime().GetVersion(); r.matchesCompat(def) { + if def := c.GetDefaultRuntime(); r.matchesCompat(def.GetVersion()) { return def } for _, rt := range c.GetRestorableRuntimes() { if r.matchesCompat(rt.GetVersion()) { - return rt.GetVersion() + return rt } } return nil } // MatchingRuntime returns the runtime from status that can restore the -// snapshot, as a stamp: the snapshot's class with the version matchingCompat -// finds, or nil when it finds none. A requirement of nothing returns -// ColdBootRuntime. +// snapshot, as a stamp: the snapshot's class with the version of the runtime +// matchingRuntime finds, or nil when it finds none. A requirement of nothing +// returns ColdBootRuntime. func (r Requirement) MatchingRuntime(status *ateapi.WorkerStatus) *ateapi.ActorSandboxRuntime { if r.snap == nil { return ColdBootRuntime(status) } - if compat := r.matchingCompat(status); compat != nil { - return runtimeOf(r.snap.GetSandboxClass(), compat) + if rt := r.matchingRuntime(status); rt != nil { + return runtimeOf(r.snap.GetSandboxClass(), rt.GetVersion()) } return nil } @@ -227,7 +227,22 @@ func (r Requirement) MatchingRuntime(status *ateapi.WorkerStatus) *ateapi.ActorS // WorkerMatches reports whether a worker with status can restore the // snapshot. A requirement of nothing is met by any worker. func (r Requirement) WorkerMatches(status *ateapi.WorkerStatus) bool { - return r.snap == nil || r.matchingCompat(status) != nil + return r.snap == nil || r.matchingRuntime(status) != nil +} + +// WorkerRuntimeFor returns the worker's runtime that RuntimeFor stamps, as the +// worker reported it, or nil before the worker has registered. +func (r Requirement) WorkerRuntimeFor(status *ateapi.WorkerStatus) *ateapi.WorkerSandboxRuntime { + if r.snap != nil { + if rt := r.matchingRuntime(status); rt != nil { + return rt + } + } + classes := status.GetSandboxClasses() + if len(classes) == 0 { + return nil + } + return classes[0].GetDefaultRuntime() } // RuntimeFor returns the runtime a worker with status runs an actor under diff --git a/internal/hardware/hardware_test.go b/internal/hardware/hardware_test.go index 0465917bc1..6569c74943 100644 --- a/internal/hardware/hardware_test.go +++ b/internal/hardware/hardware_test.go @@ -320,3 +320,46 @@ func TestMatchingRuntimeAndWorkerMatches(t *testing.T) { } }) } + +func TestWorkerRuntimeFor(t *testing.T) { + compat := func(hash string) *ateapi.VersionedSandboxCompat { + return &ateapi.VersionedSandboxCompat{SchemaVersion: SchemaVersionV1, Attributes: []*ateapi.AttributeEntry{ + {Key: AttrArchitecture, Value: "amd64"}, + {Key: AttrSandboxBundleHash, Value: hash}, + }} + } + def := &ateapi.WorkerSandboxRuntime{Version: compat("new"), ActorOverhead: &ateapi.RuntimeOverhead{MemoryBytes: 4 << 20}} + old := &ateapi.WorkerSandboxRuntime{Version: compat("old"), ActorOverhead: &ateapi.RuntimeOverhead{MemoryBytes: 16 << 20}} + status := &ateapi.WorkerStatus{SandboxClasses: []*ateapi.WorkerSandboxClass{{ + SandboxClass: "microvm", + DefaultRuntime: def, + RestorableRuntimes: []*ateapi.WorkerSandboxRuntime{old}, + }}} + stamp := func(c *ateapi.VersionedSandboxCompat) *ateapi.ActorSandboxRuntime { + return &ateapi.ActorSandboxRuntime{SandboxClass: "microvm", Version: c} + } + for _, tc := range []struct { + name string + snap *ateapi.ActorSandboxRuntime + want *ateapi.WorkerSandboxRuntime + }{ + {name: "cold boot runs the default", want: def}, + {name: "a restore runs the matching runtime", snap: stamp(compat("old")), want: old}, + {name: "a snapshot nothing restores cold boots on the default", snap: stamp(compat("gone")), want: def}, + } { + t.Run(tc.name, func(t *testing.T) { + r := NewRequirement(tc.snap) + got := r.WorkerRuntimeFor(status) + if got != tc.want { + t.Errorf("WorkerRuntimeFor() = %v, want %v", got, tc.want) + } + // it is the runtime RuntimeFor stamps + if stamped := r.RuntimeFor(status); !proto.Equal(stamped.GetVersion(), got.GetVersion()) { + t.Errorf("RuntimeFor() = %v, which is not WorkerRuntimeFor()'s %v", stamped, got) + } + }) + } + if got := NewRequirement(nil).WorkerRuntimeFor(&ateapi.WorkerStatus{}); got != nil { + t.Errorf("WorkerRuntimeFor() before registration = %v, want nil", got) + } +} diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 70f40de83c..c674b1fde6 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -615,7 +615,12 @@ type WorkerSandboxRuntime struct { // runtime. // // +k8s:required - Version *VersionedSandboxCompat `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + Version *VersionedSandboxCompat `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + // The resource overhead incurred by the worker for each actor running under + // this runtime, on top of the actor's stated limits. + // + // +k8s:optional + ActorOverhead *RuntimeOverhead `protobuf:"bytes,2,opt,name=actor_overhead,json=actorOverhead,proto3" json:"actor_overhead,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -657,6 +662,78 @@ func (x *WorkerSandboxRuntime) GetVersion() *VersionedSandboxCompat { return nil } +func (x *WorkerSandboxRuntime) GetActorOverhead() *RuntimeOverhead { + if x != nil { + return x.ActorOverhead + } + return nil +} + +// RuntimeOverhead is the memory an actor uses on its worker outside its own +// limits. It mirrors the control plane's RuntimeOverhead. +type RuntimeOverhead struct { + state protoimpl.MessageState `protogen:"open.v1"` + // memory_bytes is a fixed amount of memory for each actor. + // + // +k8s:optional + // +k8s:minimum=0 + // +k8s:maximum=1099511627776 # 1TiB + MemoryBytes int64 `protobuf:"varint,1,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` + // memory_bytes_per_gib is memory per GiB of the actor's memory limit, in + // proportion and rounded down to a byte. + // + // +k8s:optional + // +k8s:minimum=0 + // +k8s:maximum=1073741824 # 1GiB + MemoryBytesPerGib int64 `protobuf:"varint,2,opt,name=memory_bytes_per_gib,json=memoryBytesPerGib,proto3" json:"memory_bytes_per_gib,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RuntimeOverhead) Reset() { + *x = RuntimeOverhead{} + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RuntimeOverhead) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RuntimeOverhead) ProtoMessage() {} + +func (x *RuntimeOverhead) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RuntimeOverhead.ProtoReflect.Descriptor instead. +func (*RuntimeOverhead) Descriptor() ([]byte, []int) { + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{7} +} + +func (x *RuntimeOverhead) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + +func (x *RuntimeOverhead) GetMemoryBytesPerGib() int64 { + if x != nil { + return x.MemoryBytesPerGib + } + return 0 +} + // ActorSandboxRuntime is the runtime an actor runs under, as named on Run and // Restore. It mirrors the control plane's ActorSandboxRuntime and is validated by // the same rules. @@ -678,7 +755,7 @@ type ActorSandboxRuntime struct { func (x *ActorSandboxRuntime) Reset() { *x = ActorSandboxRuntime{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[7] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -690,7 +767,7 @@ func (x *ActorSandboxRuntime) String() string { func (*ActorSandboxRuntime) ProtoMessage() {} func (x *ActorSandboxRuntime) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[7] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -703,7 +780,7 @@ func (x *ActorSandboxRuntime) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorSandboxRuntime.ProtoReflect.Descriptor instead. func (*ActorSandboxRuntime) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{7} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{8} } func (x *ActorSandboxRuntime) GetSandboxClass() string { @@ -753,7 +830,7 @@ type VersionedSandboxCompat struct { func (x *VersionedSandboxCompat) Reset() { *x = VersionedSandboxCompat{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[8] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -765,7 +842,7 @@ func (x *VersionedSandboxCompat) String() string { func (*VersionedSandboxCompat) ProtoMessage() {} func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[8] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -778,7 +855,7 @@ func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { // Deprecated: Use VersionedSandboxCompat.ProtoReflect.Descriptor instead. func (*VersionedSandboxCompat) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{8} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{9} } func (x *VersionedSandboxCompat) GetSchemaVersion() string { @@ -814,7 +891,7 @@ type AttributeEntry struct { func (x *AttributeEntry) Reset() { *x = AttributeEntry{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[9] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -826,7 +903,7 @@ func (x *AttributeEntry) String() string { func (*AttributeEntry) ProtoMessage() {} func (x *AttributeEntry) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[9] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -839,7 +916,7 @@ func (x *AttributeEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use AttributeEntry.ProtoReflect.Descriptor instead. func (*AttributeEntry) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{9} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{10} } func (x *AttributeEntry) GetKey() string { @@ -886,7 +963,7 @@ type WorkerResources struct { func (x *WorkerResources) Reset() { *x = WorkerResources{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[10] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -898,7 +975,7 @@ func (x *WorkerResources) String() string { func (*WorkerResources) ProtoMessage() {} func (x *WorkerResources) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[10] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -911,7 +988,7 @@ func (x *WorkerResources) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. func (*WorkerResources) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{10} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{11} } func (x *WorkerResources) GetCpuMillicores() int64 { @@ -948,7 +1025,7 @@ type ResourceRequirements struct { func (x *ResourceRequirements) Reset() { *x = ResourceRequirements{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[11] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -960,7 +1037,7 @@ func (x *ResourceRequirements) String() string { func (*ResourceRequirements) ProtoMessage() {} func (x *ResourceRequirements) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[11] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -973,7 +1050,7 @@ func (x *ResourceRequirements) ProtoReflect() protoreflect.Message { // Deprecated: Use ResourceRequirements.ProtoReflect.Descriptor instead. func (*ResourceRequirements) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{11} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{12} } func (x *ResourceRequirements) GetLimits() *Resources { @@ -1003,7 +1080,7 @@ type Resources struct { func (x *Resources) Reset() { *x = Resources{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[12] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1015,7 +1092,7 @@ func (x *Resources) String() string { func (*Resources) ProtoMessage() {} func (x *Resources) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[12] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1028,7 +1105,7 @@ func (x *Resources) ProtoReflect() protoreflect.Message { // Deprecated: Use Resources.ProtoReflect.Descriptor instead. func (*Resources) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{12} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{13} } func (x *Resources) GetCpuMillicores() int64 { @@ -1053,7 +1130,7 @@ type RegisterWorkerResponse struct { func (x *RegisterWorkerResponse) Reset() { *x = RegisterWorkerResponse{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[13] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1065,7 +1142,7 @@ func (x *RegisterWorkerResponse) String() string { func (*RegisterWorkerResponse) ProtoMessage() {} func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[13] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1078,7 +1155,7 @@ func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerResponse.ProtoReflect.Descriptor instead. func (*RegisterWorkerResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{13} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{14} } type RequestActorSuspendRequest struct { @@ -1105,7 +1182,7 @@ type RequestActorSuspendRequest struct { func (x *RequestActorSuspendRequest) Reset() { *x = RequestActorSuspendRequest{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[14] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1117,7 +1194,7 @@ func (x *RequestActorSuspendRequest) String() string { func (*RequestActorSuspendRequest) ProtoMessage() {} func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[14] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1130,7 +1207,7 @@ func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendRequest.ProtoReflect.Descriptor instead. func (*RequestActorSuspendRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{14} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{15} } func (x *RequestActorSuspendRequest) GetActorAtespace() string { @@ -1162,7 +1239,7 @@ type RequestActorSuspendResponse struct { func (x *RequestActorSuspendResponse) Reset() { *x = RequestActorSuspendResponse{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[15] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1174,7 +1251,7 @@ func (x *RequestActorSuspendResponse) String() string { func (*RequestActorSuspendResponse) ProtoMessage() {} func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[15] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1187,7 +1264,7 @@ func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendResponse.ProtoReflect.Descriptor instead. func (*RequestActorSuspendResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{15} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{16} } type ReportActorCrashedRequest struct { @@ -1214,7 +1291,7 @@ type ReportActorCrashedRequest struct { func (x *ReportActorCrashedRequest) Reset() { *x = ReportActorCrashedRequest{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[16] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1226,7 +1303,7 @@ func (x *ReportActorCrashedRequest) String() string { func (*ReportActorCrashedRequest) ProtoMessage() {} func (x *ReportActorCrashedRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[16] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1239,7 +1316,7 @@ func (x *ReportActorCrashedRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorCrashedRequest.ProtoReflect.Descriptor instead. func (*ReportActorCrashedRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{16} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{17} } func (x *ReportActorCrashedRequest) GetActorAtespace() string { @@ -1278,7 +1355,7 @@ type ReportActorCrashedResponse struct { func (x *ReportActorCrashedResponse) Reset() { *x = ReportActorCrashedResponse{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[17] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1290,7 +1367,7 @@ func (x *ReportActorCrashedResponse) String() string { func (*ReportActorCrashedResponse) ProtoMessage() {} func (x *ReportActorCrashedResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[17] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1303,7 +1380,7 @@ func (x *ReportActorCrashedResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportActorCrashedResponse.ProtoReflect.Descriptor instead. func (*ReportActorCrashedResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{17} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{18} } type GetActorEgressPolicyRequest struct { @@ -1323,7 +1400,7 @@ type GetActorEgressPolicyRequest struct { func (x *GetActorEgressPolicyRequest) Reset() { *x = GetActorEgressPolicyRequest{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[18] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1335,7 +1412,7 @@ func (x *GetActorEgressPolicyRequest) String() string { func (*GetActorEgressPolicyRequest) ProtoMessage() {} func (x *GetActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[18] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1348,7 +1425,7 @@ func (x *GetActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*GetActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{18} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{19} } func (x *GetActorEgressPolicyRequest) GetActorAtespace() string { @@ -1376,7 +1453,7 @@ type GetActorEgressPolicyResponse struct { func (x *GetActorEgressPolicyResponse) Reset() { *x = GetActorEgressPolicyResponse{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[19] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1388,7 +1465,7 @@ func (x *GetActorEgressPolicyResponse) String() string { func (*GetActorEgressPolicyResponse) ProtoMessage() {} func (x *GetActorEgressPolicyResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[19] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1401,7 +1478,7 @@ func (x *GetActorEgressPolicyResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorEgressPolicyResponse.ProtoReflect.Descriptor instead. func (*GetActorEgressPolicyResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{19} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{20} } func (x *GetActorEgressPolicyResponse) GetEgressPolicy() *EgressPolicy { @@ -1425,7 +1502,7 @@ type EgressPolicy struct { func (x *EgressPolicy) Reset() { *x = EgressPolicy{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[20] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1437,7 +1514,7 @@ func (x *EgressPolicy) String() string { func (*EgressPolicy) ProtoMessage() {} func (x *EgressPolicy) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[20] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1450,7 +1527,7 @@ func (x *EgressPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressPolicy.ProtoReflect.Descriptor instead. func (*EgressPolicy) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{20} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{21} } func (x *EgressPolicy) GetRules() []*EgressRule { @@ -1477,7 +1554,7 @@ type EgressRule struct { func (x *EgressRule) Reset() { *x = EgressRule{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[21] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1489,7 +1566,7 @@ func (x *EgressRule) String() string { func (*EgressRule) ProtoMessage() {} func (x *EgressRule) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[21] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1502,7 +1579,7 @@ func (x *EgressRule) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressRule.ProtoReflect.Descriptor instead. func (*EgressRule) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{21} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{22} } func (x *EgressRule) GetHttp() *EgressDestinations { @@ -1540,7 +1617,7 @@ type EgressDestinations struct { func (x *EgressDestinations) Reset() { *x = EgressDestinations{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[22] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1552,7 +1629,7 @@ func (x *EgressDestinations) String() string { func (*EgressDestinations) ProtoMessage() {} func (x *EgressDestinations) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[22] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1565,7 +1642,7 @@ func (x *EgressDestinations) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressDestinations.ProtoReflect.Descriptor instead. func (*EgressDestinations) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{22} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{23} } func (x *EgressDestinations) GetHostnames() []string { @@ -1596,7 +1673,7 @@ type EgressPorts struct { func (x *EgressPorts) Reset() { *x = EgressPorts{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[23] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1608,7 +1685,7 @@ func (x *EgressPorts) String() string { func (*EgressPorts) ProtoMessage() {} func (x *EgressPorts) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[23] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1621,7 +1698,7 @@ func (x *EgressPorts) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressPorts.ProtoReflect.Descriptor instead. func (*EgressPorts) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{23} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{24} } func (x *EgressPorts) GetAll() *EgressAllPorts { @@ -1647,7 +1724,7 @@ type EgressAllPorts struct { func (x *EgressAllPorts) Reset() { *x = EgressAllPorts{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[24] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1659,7 +1736,7 @@ func (x *EgressAllPorts) String() string { func (*EgressAllPorts) ProtoMessage() {} func (x *EgressAllPorts) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[24] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1672,7 +1749,7 @@ func (x *EgressAllPorts) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressAllPorts.ProtoReflect.Descriptor instead. func (*EgressAllPorts) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{24} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{25} } type MintActorCertificateRequest struct { @@ -1703,7 +1780,7 @@ type MintActorCertificateRequest struct { func (x *MintActorCertificateRequest) Reset() { *x = MintActorCertificateRequest{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[25] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1715,7 +1792,7 @@ func (x *MintActorCertificateRequest) String() string { func (*MintActorCertificateRequest) ProtoMessage() {} func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[25] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1728,7 +1805,7 @@ func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{25} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{26} } func (x *MintActorCertificateRequest) GetActorAtespace() string { @@ -1769,7 +1846,7 @@ type MintActorCertificateResponse struct { func (x *MintActorCertificateResponse) Reset() { *x = MintActorCertificateResponse{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[26] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1781,7 +1858,7 @@ func (x *MintActorCertificateResponse) String() string { func (*MintActorCertificateResponse) ProtoMessage() {} func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[26] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1794,7 +1871,7 @@ func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{26} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{27} } func (x *MintActorCertificateResponse) GetActorCertificates() [][]byte { @@ -1840,7 +1917,7 @@ type TerminateRequest struct { func (x *TerminateRequest) Reset() { *x = TerminateRequest{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[27] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1852,7 +1929,7 @@ func (x *TerminateRequest) String() string { func (*TerminateRequest) ProtoMessage() {} func (x *TerminateRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[27] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1865,7 +1942,7 @@ func (x *TerminateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use TerminateRequest.ProtoReflect.Descriptor instead. func (*TerminateRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{27} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{28} } func (x *TerminateRequest) GetWorkerPodUid() string { @@ -1925,7 +2002,7 @@ type TerminateResponse struct { func (x *TerminateResponse) Reset() { *x = TerminateResponse{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[28] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1937,7 +2014,7 @@ func (x *TerminateResponse) String() string { func (*TerminateResponse) ProtoMessage() {} func (x *TerminateResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[28] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1950,7 +2027,7 @@ func (x *TerminateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use TerminateResponse.ProtoReflect.Descriptor instead. func (*TerminateResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{28} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{29} } type RunRequest struct { @@ -2002,7 +2079,7 @@ type RunRequest struct { func (x *RunRequest) Reset() { *x = RunRequest{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[29] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2014,7 +2091,7 @@ func (x *RunRequest) String() string { func (*RunRequest) ProtoMessage() {} func (x *RunRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[29] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2027,7 +2104,7 @@ func (x *RunRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RunRequest.ProtoReflect.Descriptor instead. func (*RunRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{29} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{30} } func (x *RunRequest) GetWorkerPodUid() string { @@ -2116,7 +2193,7 @@ type EgressGateway struct { func (x *EgressGateway) Reset() { *x = EgressGateway{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[30] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2128,7 +2205,7 @@ func (x *EgressGateway) String() string { func (*EgressGateway) ProtoMessage() {} func (x *EgressGateway) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[30] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2141,7 +2218,7 @@ func (x *EgressGateway) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressGateway.ProtoReflect.Descriptor instead. func (*EgressGateway) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{30} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{31} } func (x *EgressGateway) GetAddress() string { @@ -2172,7 +2249,7 @@ type WorkloadSpec struct { func (x *WorkloadSpec) Reset() { *x = WorkloadSpec{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[31] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2184,7 +2261,7 @@ func (x *WorkloadSpec) String() string { func (*WorkloadSpec) ProtoMessage() {} func (x *WorkloadSpec) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[31] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2197,7 +2274,7 @@ func (x *WorkloadSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkloadSpec.ProtoReflect.Descriptor instead. func (*WorkloadSpec) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{31} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{32} } func (x *WorkloadSpec) GetContainers() []*Container { @@ -2222,7 +2299,7 @@ type DurableDirVolumeSource struct { func (x *DurableDirVolumeSource) Reset() { *x = DurableDirVolumeSource{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[32] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2234,7 +2311,7 @@ func (x *DurableDirVolumeSource) String() string { func (*DurableDirVolumeSource) ProtoMessage() {} func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[32] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2247,7 +2324,7 @@ func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use DurableDirVolumeSource.ProtoReflect.Descriptor instead. func (*DurableDirVolumeSource) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{32} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{33} } type ExternalVolumeSource struct { @@ -2281,7 +2358,7 @@ type ExternalVolumeSource struct { func (x *ExternalVolumeSource) Reset() { *x = ExternalVolumeSource{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[33] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2293,7 +2370,7 @@ func (x *ExternalVolumeSource) String() string { func (*ExternalVolumeSource) ProtoMessage() {} func (x *ExternalVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[33] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2306,7 +2383,7 @@ func (x *ExternalVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalVolumeSource.ProtoReflect.Descriptor instead. func (*ExternalVolumeSource) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{33} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{34} } func (x *ExternalVolumeSource) GetStorageVolumeId() string { @@ -2351,7 +2428,7 @@ type ImageVolumeSource struct { func (x *ImageVolumeSource) Reset() { *x = ImageVolumeSource{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[34] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2363,7 +2440,7 @@ func (x *ImageVolumeSource) String() string { func (*ImageVolumeSource) ProtoMessage() {} func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[34] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2376,7 +2453,7 @@ func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ImageVolumeSource.ProtoReflect.Descriptor instead. func (*ImageVolumeSource) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{34} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{35} } func (x *ImageVolumeSource) GetReference() string { @@ -2408,7 +2485,7 @@ type ActorMetadataItem struct { func (x *ActorMetadataItem) Reset() { *x = ActorMetadataItem{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[35] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2420,7 +2497,7 @@ func (x *ActorMetadataItem) String() string { func (*ActorMetadataItem) ProtoMessage() {} func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[35] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2433,7 +2510,7 @@ func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataItem.ProtoReflect.Descriptor instead. func (*ActorMetadataItem) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{35} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{36} } func (x *ActorMetadataItem) GetField() ActorMetadataField { @@ -2469,7 +2546,7 @@ type ActorMetadataDataSource struct { func (x *ActorMetadataDataSource) Reset() { *x = ActorMetadataDataSource{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[36] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2481,7 +2558,7 @@ func (x *ActorMetadataDataSource) String() string { func (*ActorMetadataDataSource) ProtoMessage() {} func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[36] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2494,7 +2571,7 @@ func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataDataSource.ProtoReflect.Descriptor instead. func (*ActorMetadataDataSource) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{36} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{37} } func (x *ActorMetadataDataSource) GetItems() []*ActorMetadataItem { @@ -2528,7 +2605,7 @@ type TrustBundleDataSource struct { func (x *TrustBundleDataSource) Reset() { *x = TrustBundleDataSource{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[37] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2540,7 +2617,7 @@ func (x *TrustBundleDataSource) String() string { func (*TrustBundleDataSource) ProtoMessage() {} func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[37] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2553,7 +2630,7 @@ func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use TrustBundleDataSource.ProtoReflect.Descriptor instead. func (*TrustBundleDataSource) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{37} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{38} } func (x *TrustBundleDataSource) GetPath() string { @@ -2588,7 +2665,7 @@ type SystemInfoDataSource struct { func (x *SystemInfoDataSource) Reset() { *x = SystemInfoDataSource{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[38] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2600,7 +2677,7 @@ func (x *SystemInfoDataSource) String() string { func (*SystemInfoDataSource) ProtoMessage() {} func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[38] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2613,7 +2690,7 @@ func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoDataSource.ProtoReflect.Descriptor instead. func (*SystemInfoDataSource) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{38} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{39} } func (x *SystemInfoDataSource) GetActorMetadata() *ActorMetadataDataSource { @@ -2646,7 +2723,7 @@ type SystemInfoVolumeSource struct { func (x *SystemInfoVolumeSource) Reset() { *x = SystemInfoVolumeSource{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[39] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2658,7 +2735,7 @@ func (x *SystemInfoVolumeSource) String() string { func (*SystemInfoVolumeSource) ProtoMessage() {} func (x *SystemInfoVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[39] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2671,7 +2748,7 @@ func (x *SystemInfoVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoVolumeSource.ProtoReflect.Descriptor instead. func (*SystemInfoVolumeSource) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{39} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{40} } func (x *SystemInfoVolumeSource) GetDataSources() []*SystemInfoDataSource { @@ -2707,7 +2784,7 @@ type Volume struct { func (x *Volume) Reset() { *x = Volume{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[40] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2719,7 +2796,7 @@ func (x *Volume) String() string { func (*Volume) ProtoMessage() {} func (x *Volume) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[40] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2732,7 +2809,7 @@ func (x *Volume) ProtoReflect() protoreflect.Message { // Deprecated: Use Volume.ProtoReflect.Descriptor instead. func (*Volume) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{40} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{41} } func (x *Volume) GetName() string { @@ -2789,7 +2866,7 @@ type VolumeMount struct { func (x *VolumeMount) Reset() { *x = VolumeMount{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[41] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2801,7 +2878,7 @@ func (x *VolumeMount) String() string { func (*VolumeMount) ProtoMessage() {} func (x *VolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[41] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2814,7 +2891,7 @@ func (x *VolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. func (*VolumeMount) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{41} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{42} } func (x *VolumeMount) GetName() string { @@ -2879,7 +2956,7 @@ type Container struct { func (x *Container) Reset() { *x = Container{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[42] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2891,7 +2968,7 @@ func (x *Container) String() string { func (*Container) ProtoMessage() {} func (x *Container) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[42] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2904,7 +2981,7 @@ func (x *Container) ProtoReflect() protoreflect.Message { // Deprecated: Use Container.ProtoReflect.Descriptor instead. func (*Container) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{42} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{43} } func (x *Container) GetName() string { @@ -2981,7 +3058,7 @@ type SecurityContext struct { func (x *SecurityContext) Reset() { *x = SecurityContext{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[43] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2993,7 +3070,7 @@ func (x *SecurityContext) String() string { func (*SecurityContext) ProtoMessage() {} func (x *SecurityContext) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[43] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3006,7 +3083,7 @@ func (x *SecurityContext) ProtoReflect() protoreflect.Message { // Deprecated: Use SecurityContext.ProtoReflect.Descriptor instead. func (*SecurityContext) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{43} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{44} } func (x *SecurityContext) GetCapabilities() *Capabilities { @@ -3036,7 +3113,7 @@ type Capabilities struct { func (x *Capabilities) Reset() { *x = Capabilities{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[44] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3048,7 +3125,7 @@ func (x *Capabilities) String() string { func (*Capabilities) ProtoMessage() {} func (x *Capabilities) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[44] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3061,7 +3138,7 @@ func (x *Capabilities) ProtoReflect() protoreflect.Message { // Deprecated: Use Capabilities.ProtoReflect.Descriptor instead. func (*Capabilities) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{44} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{45} } func (x *Capabilities) GetAdd() []string { @@ -3095,7 +3172,7 @@ type EnvVar struct { func (x *EnvVar) Reset() { *x = EnvVar{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[45] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3107,7 +3184,7 @@ func (x *EnvVar) String() string { func (*EnvVar) ProtoMessage() {} func (x *EnvVar) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[45] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3120,7 +3197,7 @@ func (x *EnvVar) ProtoReflect() protoreflect.Message { // Deprecated: Use EnvVar.ProtoReflect.Descriptor instead. func (*EnvVar) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{45} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{46} } func (x *EnvVar) GetName() string { @@ -3155,7 +3232,7 @@ type WakeupProbe struct { func (x *WakeupProbe) Reset() { *x = WakeupProbe{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[46] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3167,7 +3244,7 @@ func (x *WakeupProbe) String() string { func (*WakeupProbe) ProtoMessage() {} func (x *WakeupProbe) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[46] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[47] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3180,7 +3257,7 @@ func (x *WakeupProbe) ProtoReflect() protoreflect.Message { // Deprecated: Use WakeupProbe.ProtoReflect.Descriptor instead. func (*WakeupProbe) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{46} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{47} } func (x *WakeupProbe) GetHttpGet() *HttpGetAction { @@ -3218,7 +3295,7 @@ type HttpGetAction struct { func (x *HttpGetAction) Reset() { *x = HttpGetAction{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[47] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3230,7 +3307,7 @@ func (x *HttpGetAction) String() string { func (*HttpGetAction) ProtoMessage() {} func (x *HttpGetAction) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[47] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[48] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3243,7 +3320,7 @@ func (x *HttpGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpGetAction.ProtoReflect.Descriptor instead. func (*HttpGetAction) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{47} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{48} } func (x *HttpGetAction) GetPath() string { @@ -3271,7 +3348,7 @@ type RunResponse struct { func (x *RunResponse) Reset() { *x = RunResponse{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[48] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3283,7 +3360,7 @@ func (x *RunResponse) String() string { func (*RunResponse) ProtoMessage() {} func (x *RunResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[48] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[49] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3296,7 +3373,7 @@ func (x *RunResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RunResponse.ProtoReflect.Descriptor instead. func (*RunResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{48} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{49} } func (x *RunResponse) GetWorkerEpoch() int64 { @@ -3329,7 +3406,7 @@ type ObjectStorage struct { func (x *ObjectStorage) Reset() { *x = ObjectStorage{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[49] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3341,7 +3418,7 @@ func (x *ObjectStorage) String() string { func (*ObjectStorage) ProtoMessage() {} func (x *ObjectStorage) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[49] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[50] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3354,7 +3431,7 @@ func (x *ObjectStorage) ProtoReflect() protoreflect.Message { // Deprecated: Use ObjectStorage.ProtoReflect.Descriptor instead. func (*ObjectStorage) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{49} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{50} } func (x *ObjectStorage) GetSnapshotUri() string { @@ -3415,7 +3492,7 @@ type Snapshot struct { func (x *Snapshot) Reset() { *x = Snapshot{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[50] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3427,7 +3504,7 @@ func (x *Snapshot) String() string { func (*Snapshot) ProtoMessage() {} func (x *Snapshot) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[50] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[51] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3440,7 +3517,7 @@ func (x *Snapshot) ProtoReflect() protoreflect.Message { // Deprecated: Use Snapshot.ProtoReflect.Descriptor instead. func (*Snapshot) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{50} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{51} } func (x *Snapshot) GetSnapshotUuid() string { @@ -3487,7 +3564,7 @@ type SnapshotReport struct { func (x *SnapshotReport) Reset() { *x = SnapshotReport{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[51] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3499,7 +3576,7 @@ func (x *SnapshotReport) String() string { func (*SnapshotReport) ProtoMessage() {} func (x *SnapshotReport) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[51] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[52] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3512,7 +3589,7 @@ func (x *SnapshotReport) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotReport.ProtoReflect.Descriptor instead. func (*SnapshotReport) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{51} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{52} } func (x *SnapshotReport) GetSnapshot() *Snapshot { @@ -3569,7 +3646,7 @@ type ActorSnapshotState struct { func (x *ActorSnapshotState) Reset() { *x = ActorSnapshotState{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[52] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3581,7 +3658,7 @@ func (x *ActorSnapshotState) String() string { func (*ActorSnapshotState) ProtoMessage() {} func (x *ActorSnapshotState) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[52] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[53] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3594,7 +3671,7 @@ func (x *ActorSnapshotState) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorSnapshotState.ProtoReflect.Descriptor instead. func (*ActorSnapshotState) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{52} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{53} } func (x *ActorSnapshotState) GetLatestLocal() *SnapshotReport { @@ -3638,7 +3715,7 @@ type UploadConfig struct { func (x *UploadConfig) Reset() { *x = UploadConfig{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[53] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3650,7 +3727,7 @@ func (x *UploadConfig) String() string { func (*UploadConfig) ProtoMessage() {} func (x *UploadConfig) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[53] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[54] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3663,7 +3740,7 @@ func (x *UploadConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use UploadConfig.ProtoReflect.Descriptor instead. func (*UploadConfig) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{53} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{54} } func (x *UploadConfig) GetMode() UploadMode { @@ -3725,7 +3802,7 @@ type CheckpointRequest struct { func (x *CheckpointRequest) Reset() { *x = CheckpointRequest{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[54] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3737,7 +3814,7 @@ func (x *CheckpointRequest) String() string { func (*CheckpointRequest) ProtoMessage() {} func (x *CheckpointRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[54] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[55] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3750,7 +3827,7 @@ func (x *CheckpointRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointRequest.ProtoReflect.Descriptor instead. func (*CheckpointRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{54} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{55} } func (x *CheckpointRequest) GetWorkerPodUid() string { @@ -3834,7 +3911,7 @@ type CheckpointResponse struct { func (x *CheckpointResponse) Reset() { *x = CheckpointResponse{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[55] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3846,7 +3923,7 @@ func (x *CheckpointResponse) String() string { func (*CheckpointResponse) ProtoMessage() {} func (x *CheckpointResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[55] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[56] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3859,7 +3936,7 @@ func (x *CheckpointResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointResponse.ProtoReflect.Descriptor instead. func (*CheckpointResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{55} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{56} } func (x *CheckpointResponse) GetState() *ActorSnapshotState { @@ -3933,7 +4010,7 @@ type RestoreRequest struct { func (x *RestoreRequest) Reset() { *x = RestoreRequest{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[56] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3945,7 +4022,7 @@ func (x *RestoreRequest) String() string { func (*RestoreRequest) ProtoMessage() {} func (x *RestoreRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[56] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[57] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3958,7 +4035,7 @@ func (x *RestoreRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreRequest.ProtoReflect.Descriptor instead. func (*RestoreRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{56} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{57} } func (x *RestoreRequest) GetWorkerPodUid() string { @@ -4049,7 +4126,7 @@ type RestoreResponse struct { func (x *RestoreResponse) Reset() { *x = RestoreResponse{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[57] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4061,7 +4138,7 @@ func (x *RestoreResponse) String() string { func (*RestoreResponse) ProtoMessage() {} func (x *RestoreResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[57] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[58] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4074,7 +4151,7 @@ func (x *RestoreResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreResponse.ProtoReflect.Descriptor instead. func (*RestoreResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{57} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{58} } func (x *RestoreResponse) GetWorkerEpoch() int64 { @@ -4114,7 +4191,7 @@ type PromoteSnapshotRequest struct { func (x *PromoteSnapshotRequest) Reset() { *x = PromoteSnapshotRequest{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[58] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4126,7 +4203,7 @@ func (x *PromoteSnapshotRequest) String() string { func (*PromoteSnapshotRequest) ProtoMessage() {} func (x *PromoteSnapshotRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[58] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[59] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4139,7 +4216,7 @@ func (x *PromoteSnapshotRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PromoteSnapshotRequest.ProtoReflect.Descriptor instead. func (*PromoteSnapshotRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{58} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{59} } func (x *PromoteSnapshotRequest) GetAtespace() string { @@ -4195,7 +4272,7 @@ type PromoteSnapshotResponse struct { func (x *PromoteSnapshotResponse) Reset() { *x = PromoteSnapshotResponse{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[59] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4207,7 +4284,7 @@ func (x *PromoteSnapshotResponse) String() string { func (*PromoteSnapshotResponse) ProtoMessage() {} func (x *PromoteSnapshotResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[59] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[60] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4220,7 +4297,7 @@ func (x *PromoteSnapshotResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PromoteSnapshotResponse.ProtoReflect.Descriptor instead. func (*PromoteSnapshotResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{59} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{60} } func (x *PromoteSnapshotResponse) GetState() *ActorSnapshotState { @@ -4249,7 +4326,7 @@ type DetachActorRequest struct { func (x *DetachActorRequest) Reset() { *x = DetachActorRequest{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[60] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4261,7 +4338,7 @@ func (x *DetachActorRequest) String() string { func (*DetachActorRequest) ProtoMessage() {} func (x *DetachActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[60] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[61] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4274,7 +4351,7 @@ func (x *DetachActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachActorRequest.ProtoReflect.Descriptor instead. func (*DetachActorRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{60} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{61} } func (x *DetachActorRequest) GetAtespace() string { @@ -4310,7 +4387,7 @@ type DetachActorResponse struct { func (x *DetachActorResponse) Reset() { *x = DetachActorResponse{} - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[61] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4322,7 +4399,7 @@ func (x *DetachActorResponse) String() string { func (*DetachActorResponse) ProtoMessage() {} func (x *DetachActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[61] + mi := &file_internal_proto_ateletpb_atelet_proto_msgTypes[62] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4335,7 +4412,7 @@ func (x *DetachActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachActorResponse.ProtoReflect.Descriptor instead. func (*DetachActorResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{61} + return file_internal_proto_ateletpb_atelet_proto_rawDescGZIP(), []int{62} } func (x *DetachActorResponse) GetState() *ActorSnapshotState { @@ -4364,9 +4441,13 @@ const file_internal_proto_ateletpb_atelet_proto_rawDesc = "" + "\x12WorkerSandboxClass\x12#\n" + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x12E\n" + "\x0fdefault_runtime\x18\x02 \x01(\v2\x1c.atelet.WorkerSandboxRuntimeR\x0edefaultRuntime\x12M\n" + - "\x13restorable_runtimes\x18\x03 \x03(\v2\x1c.atelet.WorkerSandboxRuntimeR\x12restorableRuntimes\"P\n" + + "\x13restorable_runtimes\x18\x03 \x03(\v2\x1c.atelet.WorkerSandboxRuntimeR\x12restorableRuntimes\"\x90\x01\n" + "\x14WorkerSandboxRuntime\x128\n" + - "\aversion\x18\x01 \x01(\v2\x1e.atelet.VersionedSandboxCompatR\aversion\"t\n" + + "\aversion\x18\x01 \x01(\v2\x1e.atelet.VersionedSandboxCompatR\aversion\x12>\n" + + "\x0eactor_overhead\x18\x02 \x01(\v2\x17.atelet.RuntimeOverheadR\ractorOverhead\"e\n" + + "\x0fRuntimeOverhead\x12!\n" + + "\fmemory_bytes\x18\x01 \x01(\x03R\vmemoryBytes\x12/\n" + + "\x14memory_bytes_per_gib\x18\x02 \x01(\x03R\x11memoryBytesPerGib\"t\n" + "\x13ActorSandboxRuntime\x12#\n" + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x128\n" + "\aversion\x18\x02 \x01(\v2\x1e.atelet.VersionedSandboxCompatR\aversion\"w\n" + @@ -4644,7 +4725,7 @@ func file_internal_proto_ateletpb_atelet_proto_rawDescGZIP() []byte { } var file_internal_proto_ateletpb_atelet_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_internal_proto_ateletpb_atelet_proto_msgTypes = make([]protoimpl.MessageInfo, 64) +var file_internal_proto_ateletpb_atelet_proto_msgTypes = make([]protoimpl.MessageInfo, 65) var file_internal_proto_ateletpb_atelet_proto_goTypes = []any{ (ActorMetadataField)(0), // 0: atelet.ActorMetadataField (SnapshotFidelity)(0), // 1: atelet.SnapshotFidelity @@ -4657,160 +4738,162 @@ var file_internal_proto_ateletpb_atelet_proto_goTypes = []any{ (*SetWorkerSandboxRuntimesResponse)(nil), // 8: atelet.SetWorkerSandboxRuntimesResponse (*WorkerSandboxClass)(nil), // 9: atelet.WorkerSandboxClass (*WorkerSandboxRuntime)(nil), // 10: atelet.WorkerSandboxRuntime - (*ActorSandboxRuntime)(nil), // 11: atelet.ActorSandboxRuntime - (*VersionedSandboxCompat)(nil), // 12: atelet.VersionedSandboxCompat - (*AttributeEntry)(nil), // 13: atelet.AttributeEntry - (*WorkerResources)(nil), // 14: atelet.WorkerResources - (*ResourceRequirements)(nil), // 15: atelet.ResourceRequirements - (*Resources)(nil), // 16: atelet.Resources - (*RegisterWorkerResponse)(nil), // 17: atelet.RegisterWorkerResponse - (*RequestActorSuspendRequest)(nil), // 18: atelet.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 19: atelet.RequestActorSuspendResponse - (*ReportActorCrashedRequest)(nil), // 20: atelet.ReportActorCrashedRequest - (*ReportActorCrashedResponse)(nil), // 21: atelet.ReportActorCrashedResponse - (*GetActorEgressPolicyRequest)(nil), // 22: atelet.GetActorEgressPolicyRequest - (*GetActorEgressPolicyResponse)(nil), // 23: atelet.GetActorEgressPolicyResponse - (*EgressPolicy)(nil), // 24: atelet.EgressPolicy - (*EgressRule)(nil), // 25: atelet.EgressRule - (*EgressDestinations)(nil), // 26: atelet.EgressDestinations - (*EgressPorts)(nil), // 27: atelet.EgressPorts - (*EgressAllPorts)(nil), // 28: atelet.EgressAllPorts - (*MintActorCertificateRequest)(nil), // 29: atelet.MintActorCertificateRequest - (*MintActorCertificateResponse)(nil), // 30: atelet.MintActorCertificateResponse - (*TerminateRequest)(nil), // 31: atelet.TerminateRequest - (*TerminateResponse)(nil), // 32: atelet.TerminateResponse - (*RunRequest)(nil), // 33: atelet.RunRequest - (*EgressGateway)(nil), // 34: atelet.EgressGateway - (*WorkloadSpec)(nil), // 35: atelet.WorkloadSpec - (*DurableDirVolumeSource)(nil), // 36: atelet.DurableDirVolumeSource - (*ExternalVolumeSource)(nil), // 37: atelet.ExternalVolumeSource - (*ImageVolumeSource)(nil), // 38: atelet.ImageVolumeSource - (*ActorMetadataItem)(nil), // 39: atelet.ActorMetadataItem - (*ActorMetadataDataSource)(nil), // 40: atelet.ActorMetadataDataSource - (*TrustBundleDataSource)(nil), // 41: atelet.TrustBundleDataSource - (*SystemInfoDataSource)(nil), // 42: atelet.SystemInfoDataSource - (*SystemInfoVolumeSource)(nil), // 43: atelet.SystemInfoVolumeSource - (*Volume)(nil), // 44: atelet.Volume - (*VolumeMount)(nil), // 45: atelet.VolumeMount - (*Container)(nil), // 46: atelet.Container - (*SecurityContext)(nil), // 47: atelet.SecurityContext - (*Capabilities)(nil), // 48: atelet.Capabilities - (*EnvVar)(nil), // 49: atelet.EnvVar - (*WakeupProbe)(nil), // 50: atelet.WakeupProbe - (*HttpGetAction)(nil), // 51: atelet.HttpGetAction - (*RunResponse)(nil), // 52: atelet.RunResponse - (*ObjectStorage)(nil), // 53: atelet.ObjectStorage - (*Snapshot)(nil), // 54: atelet.Snapshot - (*SnapshotReport)(nil), // 55: atelet.SnapshotReport - (*ActorSnapshotState)(nil), // 56: atelet.ActorSnapshotState - (*UploadConfig)(nil), // 57: atelet.UploadConfig - (*CheckpointRequest)(nil), // 58: atelet.CheckpointRequest - (*CheckpointResponse)(nil), // 59: atelet.CheckpointResponse - (*RestoreRequest)(nil), // 60: atelet.RestoreRequest - (*RestoreResponse)(nil), // 61: atelet.RestoreResponse - (*PromoteSnapshotRequest)(nil), // 62: atelet.PromoteSnapshotRequest - (*PromoteSnapshotResponse)(nil), // 63: atelet.PromoteSnapshotResponse - (*DetachActorRequest)(nil), // 64: atelet.DetachActorRequest - (*DetachActorResponse)(nil), // 65: atelet.DetachActorResponse - nil, // 66: atelet.ExternalVolumeSource.VolumeContextEntry - nil, // 67: atelet.ExternalVolumeSource.PublishContextEntry - (*timestamppb.Timestamp)(nil), // 68: google.protobuf.Timestamp + (*RuntimeOverhead)(nil), // 11: atelet.RuntimeOverhead + (*ActorSandboxRuntime)(nil), // 12: atelet.ActorSandboxRuntime + (*VersionedSandboxCompat)(nil), // 13: atelet.VersionedSandboxCompat + (*AttributeEntry)(nil), // 14: atelet.AttributeEntry + (*WorkerResources)(nil), // 15: atelet.WorkerResources + (*ResourceRequirements)(nil), // 16: atelet.ResourceRequirements + (*Resources)(nil), // 17: atelet.Resources + (*RegisterWorkerResponse)(nil), // 18: atelet.RegisterWorkerResponse + (*RequestActorSuspendRequest)(nil), // 19: atelet.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 20: atelet.RequestActorSuspendResponse + (*ReportActorCrashedRequest)(nil), // 21: atelet.ReportActorCrashedRequest + (*ReportActorCrashedResponse)(nil), // 22: atelet.ReportActorCrashedResponse + (*GetActorEgressPolicyRequest)(nil), // 23: atelet.GetActorEgressPolicyRequest + (*GetActorEgressPolicyResponse)(nil), // 24: atelet.GetActorEgressPolicyResponse + (*EgressPolicy)(nil), // 25: atelet.EgressPolicy + (*EgressRule)(nil), // 26: atelet.EgressRule + (*EgressDestinations)(nil), // 27: atelet.EgressDestinations + (*EgressPorts)(nil), // 28: atelet.EgressPorts + (*EgressAllPorts)(nil), // 29: atelet.EgressAllPorts + (*MintActorCertificateRequest)(nil), // 30: atelet.MintActorCertificateRequest + (*MintActorCertificateResponse)(nil), // 31: atelet.MintActorCertificateResponse + (*TerminateRequest)(nil), // 32: atelet.TerminateRequest + (*TerminateResponse)(nil), // 33: atelet.TerminateResponse + (*RunRequest)(nil), // 34: atelet.RunRequest + (*EgressGateway)(nil), // 35: atelet.EgressGateway + (*WorkloadSpec)(nil), // 36: atelet.WorkloadSpec + (*DurableDirVolumeSource)(nil), // 37: atelet.DurableDirVolumeSource + (*ExternalVolumeSource)(nil), // 38: atelet.ExternalVolumeSource + (*ImageVolumeSource)(nil), // 39: atelet.ImageVolumeSource + (*ActorMetadataItem)(nil), // 40: atelet.ActorMetadataItem + (*ActorMetadataDataSource)(nil), // 41: atelet.ActorMetadataDataSource + (*TrustBundleDataSource)(nil), // 42: atelet.TrustBundleDataSource + (*SystemInfoDataSource)(nil), // 43: atelet.SystemInfoDataSource + (*SystemInfoVolumeSource)(nil), // 44: atelet.SystemInfoVolumeSource + (*Volume)(nil), // 45: atelet.Volume + (*VolumeMount)(nil), // 46: atelet.VolumeMount + (*Container)(nil), // 47: atelet.Container + (*SecurityContext)(nil), // 48: atelet.SecurityContext + (*Capabilities)(nil), // 49: atelet.Capabilities + (*EnvVar)(nil), // 50: atelet.EnvVar + (*WakeupProbe)(nil), // 51: atelet.WakeupProbe + (*HttpGetAction)(nil), // 52: atelet.HttpGetAction + (*RunResponse)(nil), // 53: atelet.RunResponse + (*ObjectStorage)(nil), // 54: atelet.ObjectStorage + (*Snapshot)(nil), // 55: atelet.Snapshot + (*SnapshotReport)(nil), // 56: atelet.SnapshotReport + (*ActorSnapshotState)(nil), // 57: atelet.ActorSnapshotState + (*UploadConfig)(nil), // 58: atelet.UploadConfig + (*CheckpointRequest)(nil), // 59: atelet.CheckpointRequest + (*CheckpointResponse)(nil), // 60: atelet.CheckpointResponse + (*RestoreRequest)(nil), // 61: atelet.RestoreRequest + (*RestoreResponse)(nil), // 62: atelet.RestoreResponse + (*PromoteSnapshotRequest)(nil), // 63: atelet.PromoteSnapshotRequest + (*PromoteSnapshotResponse)(nil), // 64: atelet.PromoteSnapshotResponse + (*DetachActorRequest)(nil), // 65: atelet.DetachActorRequest + (*DetachActorResponse)(nil), // 66: atelet.DetachActorResponse + nil, // 67: atelet.ExternalVolumeSource.VolumeContextEntry + nil, // 68: atelet.ExternalVolumeSource.PublishContextEntry + (*timestamppb.Timestamp)(nil), // 69: google.protobuf.Timestamp } var file_internal_proto_ateletpb_atelet_proto_depIdxs = []int32{ - 14, // 0: atelet.RegisterWorkerRequest.capacity:type_name -> atelet.WorkerResources + 15, // 0: atelet.RegisterWorkerRequest.capacity:type_name -> atelet.WorkerResources 9, // 1: atelet.SetWorkerSandboxRuntimesRequest.sandbox_classes:type_name -> atelet.WorkerSandboxClass 10, // 2: atelet.WorkerSandboxClass.default_runtime:type_name -> atelet.WorkerSandboxRuntime 10, // 3: atelet.WorkerSandboxClass.restorable_runtimes:type_name -> atelet.WorkerSandboxRuntime - 12, // 4: atelet.WorkerSandboxRuntime.version:type_name -> atelet.VersionedSandboxCompat - 12, // 5: atelet.ActorSandboxRuntime.version:type_name -> atelet.VersionedSandboxCompat - 13, // 6: atelet.VersionedSandboxCompat.attributes:type_name -> atelet.AttributeEntry - 16, // 7: atelet.ResourceRequirements.limits:type_name -> atelet.Resources - 24, // 8: atelet.GetActorEgressPolicyResponse.egress_policy:type_name -> atelet.EgressPolicy - 25, // 9: atelet.EgressPolicy.rules:type_name -> atelet.EgressRule - 26, // 10: atelet.EgressRule.http:type_name -> atelet.EgressDestinations - 26, // 11: atelet.EgressRule.https:type_name -> atelet.EgressDestinations - 26, // 12: atelet.EgressRule.tls_passthrough:type_name -> atelet.EgressDestinations - 27, // 13: atelet.EgressDestinations.ports:type_name -> atelet.EgressPorts - 28, // 14: atelet.EgressPorts.all:type_name -> atelet.EgressAllPorts - 35, // 15: atelet.TerminateRequest.spec:type_name -> atelet.WorkloadSpec - 35, // 16: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec - 34, // 17: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway - 15, // 18: atelet.RunRequest.resources:type_name -> atelet.ResourceRequirements - 11, // 19: atelet.RunRequest.sandbox_runtime:type_name -> atelet.ActorSandboxRuntime - 46, // 20: atelet.WorkloadSpec.containers:type_name -> atelet.Container - 44, // 21: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume - 66, // 22: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry - 67, // 23: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry - 0, // 24: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField - 39, // 25: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem - 40, // 26: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource - 41, // 27: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource - 42, // 28: atelet.SystemInfoVolumeSource.data_sources:type_name -> atelet.SystemInfoDataSource - 36, // 29: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolumeSource - 37, // 30: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource - 43, // 31: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolumeSource - 38, // 32: atelet.Volume.image:type_name -> atelet.ImageVolumeSource - 49, // 33: atelet.Container.env:type_name -> atelet.EnvVar - 50, // 34: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe - 45, // 35: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount - 47, // 36: atelet.Container.security_context:type_name -> atelet.SecurityContext - 15, // 37: atelet.Container.resources:type_name -> atelet.ResourceRequirements - 48, // 38: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities - 51, // 39: atelet.WakeupProbe.http_get:type_name -> atelet.HttpGetAction - 2, // 40: atelet.ObjectStorage.sharing:type_name -> atelet.SnapshotSharing - 1, // 41: atelet.Snapshot.fidelity:type_name -> atelet.SnapshotFidelity - 53, // 42: atelet.Snapshot.object:type_name -> atelet.ObjectStorage - 54, // 43: atelet.SnapshotReport.snapshot:type_name -> atelet.Snapshot - 68, // 44: atelet.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp - 68, // 45: atelet.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp - 55, // 46: atelet.ActorSnapshotState.latest_local:type_name -> atelet.SnapshotReport - 55, // 47: atelet.ActorSnapshotState.uploading:type_name -> atelet.SnapshotReport - 55, // 48: atelet.ActorSnapshotState.latest_durable:type_name -> atelet.SnapshotReport - 68, // 49: atelet.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp - 3, // 50: atelet.UploadConfig.mode:type_name -> atelet.UploadMode - 35, // 51: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec - 54, // 52: atelet.CheckpointRequest.snapshot:type_name -> atelet.Snapshot - 57, // 53: atelet.CheckpointRequest.upload_config:type_name -> atelet.UploadConfig - 56, // 54: atelet.CheckpointResponse.state:type_name -> atelet.ActorSnapshotState - 35, // 55: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec - 54, // 56: atelet.RestoreRequest.snapshot:type_name -> atelet.Snapshot - 34, // 57: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway - 15, // 58: atelet.RestoreRequest.resources:type_name -> atelet.ResourceRequirements - 11, // 59: atelet.RestoreRequest.sandbox_runtime:type_name -> atelet.ActorSandboxRuntime - 54, // 60: atelet.PromoteSnapshotRequest.snapshot:type_name -> atelet.Snapshot - 56, // 61: atelet.PromoteSnapshotResponse.state:type_name -> atelet.ActorSnapshotState - 56, // 62: atelet.DetachActorResponse.state:type_name -> atelet.ActorSnapshotState - 29, // 63: atelet.WorkerSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest - 4, // 64: atelet.WorkerSupport.AcquireWorkerEpoch:input_type -> atelet.AcquireWorkerEpochRequest - 6, // 65: atelet.WorkerSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest - 7, // 66: atelet.WorkerSupport.SetWorkerSandboxRuntimes:input_type -> atelet.SetWorkerSandboxRuntimesRequest - 18, // 67: atelet.WorkerSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest - 22, // 68: atelet.WorkerSupport.GetActorEgressPolicy:input_type -> atelet.GetActorEgressPolicyRequest - 20, // 69: atelet.WorkerSupport.ReportActorCrashed:input_type -> atelet.ReportActorCrashedRequest - 33, // 70: atelet.Atelet.Run:input_type -> atelet.RunRequest - 58, // 71: atelet.Atelet.Checkpoint:input_type -> atelet.CheckpointRequest - 60, // 72: atelet.Atelet.Restore:input_type -> atelet.RestoreRequest - 62, // 73: atelet.Atelet.PromoteSnapshot:input_type -> atelet.PromoteSnapshotRequest - 64, // 74: atelet.Atelet.DetachActor:input_type -> atelet.DetachActorRequest - 31, // 75: atelet.Atelet.Terminate:input_type -> atelet.TerminateRequest - 30, // 76: atelet.WorkerSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse - 5, // 77: atelet.WorkerSupport.AcquireWorkerEpoch:output_type -> atelet.AcquireWorkerEpochResponse - 17, // 78: atelet.WorkerSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse - 8, // 79: atelet.WorkerSupport.SetWorkerSandboxRuntimes:output_type -> atelet.SetWorkerSandboxRuntimesResponse - 19, // 80: atelet.WorkerSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse - 23, // 81: atelet.WorkerSupport.GetActorEgressPolicy:output_type -> atelet.GetActorEgressPolicyResponse - 21, // 82: atelet.WorkerSupport.ReportActorCrashed:output_type -> atelet.ReportActorCrashedResponse - 52, // 83: atelet.Atelet.Run:output_type -> atelet.RunResponse - 59, // 84: atelet.Atelet.Checkpoint:output_type -> atelet.CheckpointResponse - 61, // 85: atelet.Atelet.Restore:output_type -> atelet.RestoreResponse - 63, // 86: atelet.Atelet.PromoteSnapshot:output_type -> atelet.PromoteSnapshotResponse - 65, // 87: atelet.Atelet.DetachActor:output_type -> atelet.DetachActorResponse - 32, // 88: atelet.Atelet.Terminate:output_type -> atelet.TerminateResponse - 76, // [76:89] is the sub-list for method output_type - 63, // [63:76] is the sub-list for method input_type - 63, // [63:63] is the sub-list for extension type_name - 63, // [63:63] is the sub-list for extension extendee - 0, // [0:63] is the sub-list for field type_name + 13, // 4: atelet.WorkerSandboxRuntime.version:type_name -> atelet.VersionedSandboxCompat + 11, // 5: atelet.WorkerSandboxRuntime.actor_overhead:type_name -> atelet.RuntimeOverhead + 13, // 6: atelet.ActorSandboxRuntime.version:type_name -> atelet.VersionedSandboxCompat + 14, // 7: atelet.VersionedSandboxCompat.attributes:type_name -> atelet.AttributeEntry + 17, // 8: atelet.ResourceRequirements.limits:type_name -> atelet.Resources + 25, // 9: atelet.GetActorEgressPolicyResponse.egress_policy:type_name -> atelet.EgressPolicy + 26, // 10: atelet.EgressPolicy.rules:type_name -> atelet.EgressRule + 27, // 11: atelet.EgressRule.http:type_name -> atelet.EgressDestinations + 27, // 12: atelet.EgressRule.https:type_name -> atelet.EgressDestinations + 27, // 13: atelet.EgressRule.tls_passthrough:type_name -> atelet.EgressDestinations + 28, // 14: atelet.EgressDestinations.ports:type_name -> atelet.EgressPorts + 29, // 15: atelet.EgressPorts.all:type_name -> atelet.EgressAllPorts + 36, // 16: atelet.TerminateRequest.spec:type_name -> atelet.WorkloadSpec + 36, // 17: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec + 35, // 18: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway + 16, // 19: atelet.RunRequest.resources:type_name -> atelet.ResourceRequirements + 12, // 20: atelet.RunRequest.sandbox_runtime:type_name -> atelet.ActorSandboxRuntime + 47, // 21: atelet.WorkloadSpec.containers:type_name -> atelet.Container + 45, // 22: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume + 67, // 23: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry + 68, // 24: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry + 0, // 25: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField + 40, // 26: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem + 41, // 27: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource + 42, // 28: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource + 43, // 29: atelet.SystemInfoVolumeSource.data_sources:type_name -> atelet.SystemInfoDataSource + 37, // 30: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolumeSource + 38, // 31: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource + 44, // 32: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolumeSource + 39, // 33: atelet.Volume.image:type_name -> atelet.ImageVolumeSource + 50, // 34: atelet.Container.env:type_name -> atelet.EnvVar + 51, // 35: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe + 46, // 36: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount + 48, // 37: atelet.Container.security_context:type_name -> atelet.SecurityContext + 16, // 38: atelet.Container.resources:type_name -> atelet.ResourceRequirements + 49, // 39: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities + 52, // 40: atelet.WakeupProbe.http_get:type_name -> atelet.HttpGetAction + 2, // 41: atelet.ObjectStorage.sharing:type_name -> atelet.SnapshotSharing + 1, // 42: atelet.Snapshot.fidelity:type_name -> atelet.SnapshotFidelity + 54, // 43: atelet.Snapshot.object:type_name -> atelet.ObjectStorage + 55, // 44: atelet.SnapshotReport.snapshot:type_name -> atelet.Snapshot + 69, // 45: atelet.SnapshotReport.local_commit_time:type_name -> google.protobuf.Timestamp + 69, // 46: atelet.SnapshotReport.durable_complete_time:type_name -> google.protobuf.Timestamp + 56, // 47: atelet.ActorSnapshotState.latest_local:type_name -> atelet.SnapshotReport + 56, // 48: atelet.ActorSnapshotState.uploading:type_name -> atelet.SnapshotReport + 56, // 49: atelet.ActorSnapshotState.latest_durable:type_name -> atelet.SnapshotReport + 69, // 50: atelet.ActorSnapshotState.upload_due_time:type_name -> google.protobuf.Timestamp + 3, // 51: atelet.UploadConfig.mode:type_name -> atelet.UploadMode + 36, // 52: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec + 55, // 53: atelet.CheckpointRequest.snapshot:type_name -> atelet.Snapshot + 58, // 54: atelet.CheckpointRequest.upload_config:type_name -> atelet.UploadConfig + 57, // 55: atelet.CheckpointResponse.state:type_name -> atelet.ActorSnapshotState + 36, // 56: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec + 55, // 57: atelet.RestoreRequest.snapshot:type_name -> atelet.Snapshot + 35, // 58: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway + 16, // 59: atelet.RestoreRequest.resources:type_name -> atelet.ResourceRequirements + 12, // 60: atelet.RestoreRequest.sandbox_runtime:type_name -> atelet.ActorSandboxRuntime + 55, // 61: atelet.PromoteSnapshotRequest.snapshot:type_name -> atelet.Snapshot + 57, // 62: atelet.PromoteSnapshotResponse.state:type_name -> atelet.ActorSnapshotState + 57, // 63: atelet.DetachActorResponse.state:type_name -> atelet.ActorSnapshotState + 30, // 64: atelet.WorkerSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest + 4, // 65: atelet.WorkerSupport.AcquireWorkerEpoch:input_type -> atelet.AcquireWorkerEpochRequest + 6, // 66: atelet.WorkerSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest + 7, // 67: atelet.WorkerSupport.SetWorkerSandboxRuntimes:input_type -> atelet.SetWorkerSandboxRuntimesRequest + 19, // 68: atelet.WorkerSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest + 23, // 69: atelet.WorkerSupport.GetActorEgressPolicy:input_type -> atelet.GetActorEgressPolicyRequest + 21, // 70: atelet.WorkerSupport.ReportActorCrashed:input_type -> atelet.ReportActorCrashedRequest + 34, // 71: atelet.Atelet.Run:input_type -> atelet.RunRequest + 59, // 72: atelet.Atelet.Checkpoint:input_type -> atelet.CheckpointRequest + 61, // 73: atelet.Atelet.Restore:input_type -> atelet.RestoreRequest + 63, // 74: atelet.Atelet.PromoteSnapshot:input_type -> atelet.PromoteSnapshotRequest + 65, // 75: atelet.Atelet.DetachActor:input_type -> atelet.DetachActorRequest + 32, // 76: atelet.Atelet.Terminate:input_type -> atelet.TerminateRequest + 31, // 77: atelet.WorkerSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse + 5, // 78: atelet.WorkerSupport.AcquireWorkerEpoch:output_type -> atelet.AcquireWorkerEpochResponse + 18, // 79: atelet.WorkerSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse + 8, // 80: atelet.WorkerSupport.SetWorkerSandboxRuntimes:output_type -> atelet.SetWorkerSandboxRuntimesResponse + 20, // 81: atelet.WorkerSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse + 24, // 82: atelet.WorkerSupport.GetActorEgressPolicy:output_type -> atelet.GetActorEgressPolicyResponse + 22, // 83: atelet.WorkerSupport.ReportActorCrashed:output_type -> atelet.ReportActorCrashedResponse + 53, // 84: atelet.Atelet.Run:output_type -> atelet.RunResponse + 60, // 85: atelet.Atelet.Checkpoint:output_type -> atelet.CheckpointResponse + 62, // 86: atelet.Atelet.Restore:output_type -> atelet.RestoreResponse + 64, // 87: atelet.Atelet.PromoteSnapshot:output_type -> atelet.PromoteSnapshotResponse + 66, // 88: atelet.Atelet.DetachActor:output_type -> atelet.DetachActorResponse + 33, // 89: atelet.Atelet.Terminate:output_type -> atelet.TerminateResponse + 77, // [77:90] is the sub-list for method output_type + 64, // [64:77] is the sub-list for method input_type + 64, // [64:64] is the sub-list for extension type_name + 64, // [64:64] is the sub-list for extension extendee + 0, // [0:64] is the sub-list for field type_name } func init() { file_internal_proto_ateletpb_atelet_proto_init() } @@ -4818,15 +4901,15 @@ func file_internal_proto_ateletpb_atelet_proto_init() { if File_internal_proto_ateletpb_atelet_proto != nil { return } - file_internal_proto_ateletpb_atelet_proto_msgTypes[29].OneofWrappers = []any{} - file_internal_proto_ateletpb_atelet_proto_msgTypes[56].OneofWrappers = []any{} + file_internal_proto_ateletpb_atelet_proto_msgTypes[30].OneofWrappers = []any{} + file_internal_proto_ateletpb_atelet_proto_msgTypes[57].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_proto_ateletpb_atelet_proto_rawDesc), len(file_internal_proto_ateletpb_atelet_proto_rawDesc)), NumEnums: 4, - NumMessages: 64, + NumMessages: 65, NumExtensions: 0, NumServices: 2, }, diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index f68551c7b6..9b20950f62 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -158,6 +158,31 @@ message WorkerSandboxRuntime { // // +k8s:required VersionedSandboxCompat version = 1; + + // The resource overhead incurred by the worker for each actor running under + // this runtime, on top of the actor's stated limits. + // + // +k8s:optional + RuntimeOverhead actor_overhead = 2; +} + +// RuntimeOverhead is the memory an actor uses on its worker outside its own +// limits. It mirrors the control plane's RuntimeOverhead. +message RuntimeOverhead { + // memory_bytes is a fixed amount of memory for each actor. + // + // +k8s:optional + // +k8s:minimum=0 + // +k8s:maximum=1099511627776 # 1TiB + int64 memory_bytes = 1; + + // memory_bytes_per_gib is memory per GiB of the actor's memory limit, in + // proportion and rounded down to a byte. + // + // +k8s:optional + // +k8s:minimum=0 + // +k8s:maximum=1073741824 # 1GiB + int64 memory_bytes_per_gib = 2; } // ActorSandboxRuntime is the runtime an actor runs under, as named on Run and From a54dc8ddc4555af5a3ef6d34d5187e1f982c3987 Mon Sep 17 00:00:00 2001 From: Grant McCloskey Date: Tue, 27 Oct 2026 21:34:21 +0000 Subject: [PATCH 18/53] Releases publish the envoy-dataplane image (#3252) Part of #3217. The default atenet dataplane runs envoy-dataplane in the router and egress gateways, but releases do not publish it, so a pre-built install has to use `--atenet-dataplane=agentgateway` or build from source. #2112 dropped it because building Rust under QEMU was slow. #3152 made the Dockerfile cross-compile, so the release job now builds it with buildx and no QEMU. Released Dockerfile images are an allowlist (`releasedDockerfiles`) next to the ko allowlist. The release jobs pin BuildKit by digest. Release builds take each Dockerfile image's digest from buildx's `--metadata-file`, build Dockerfile images before the ko images, ignore `DOCKER_BUILD_FLAGS`, and label every image with its source, revision and version. With every image a default install deploys now published, the release notes give the install command. After the first release that includes it, an org owner has to make the new `ghcr.io/agent-substrate/substrate/envoy-dataplane` package public. ### Validation `release.yaml` from this branch, run in my fork at 619f0d95 with a throwaway tag `v0.0.0-envoytest.1`: - [`images-dry-run`](https://github.com/MushuEE/substrate/actions/runs/40562367990), dispatched with `dry_run_images`, built all eleven release images cold on `ubuntu-latest` and passed the platform and digest checks in 11m35s, well inside the 30-minute timeout. - The [tag push run](https://github.com/MushuEE/substrate/actions/runs/40561810523) created the draft release. The images job skips itself outside agent-substrate/substrate, as designed. Earlier, at bbd86145, I ran the same publish locally against a throwaway registry, as the `images-dry-run` job does: ```sh docker run -d --rm --name ate-release-test -p 5000:5000 \ registry:3@sha256:ddf754342cfc8acc51a56d5d0ab6af06826461864460636d8bd5c546dab2a7b8 printf '[registry."localhost:5000"]\n http = true\n' > /tmp/buildkitd.toml docker buildx create --name ate-release --driver docker-container \ --driver-opt network=host --buildkitd-config /tmp/buildkitd.toml --use VERSION=v0.0.0-local.1 KO_DEFAULTPLATFORMS=linux/amd64,linux/arm64 \ go run ./cmd/ate-setup publish release-images --ko-docker-repo localhost:5000/substrate docker buildx imagetools inspect localhost:5000/substrate/envoy-dataplane:v0.0.0-local.1 docker buildx imagetools inspect localhost:5000/substrate/envoy-dataplane:v0.0.0-local.1 \ --format '{{json (index .Image "linux/amd64").Config.Labels}}' ``` - buildx cross-compiled the Rust module (`linux/amd64->arm64`), with no QEMU. - The envoy-dataplane digest from `--metadata-file` matched what the tag resolved to. - envoy-dataplane had `linux/amd64` and `linux/arm64`, and the `source` and `revision` labels. - A rerun built nothing. That run showed envoy-dataplane inheriting `org.opencontainers.image.version: "22.04"` from the Ubuntu base of `envoyproxy/envoy`. Release images now set that label to the release tag. AI use: I used Claude Code to trace the code #2112 removed, draft the change, address review comments, and script the local and fork validation. I reviewed the diff and ran the ate-setup tests and the validation above. - [x] Tests pass - [x] Docs updated --------- Signed-off-by: Grant McCloskey --- .github/scripts/release/ghcr-login.sh | 4 +- .github/workflows/release.yaml | 23 ++++++- cmd/ate-setup/README.md | 6 +- cmd/ate-setup/commands.md | 6 +- cmd/ate-setup/differences.md | 10 +-- cmd/ate-setup/internal/cmd/publish.go | 12 ++-- cmd/ate-setup/internal/images/dockerfile.go | 69 ++++++++++++------- .../internal/images/dockerfile_test.go | 38 ++++++++-- cmd/ate-setup/internal/images/images.go | 19 +++++ cmd/ate-setup/internal/images/release.go | 5 ++ cmd/ate-setup/internal/steps/env.go | 18 ++--- cmd/ate-setup/internal/steps/overlay.go | 11 ++- cmd/ate-setup/internal/steps/overlay_test.go | 12 ++-- cmd/ate-setup/internal/steps/publish.go | 38 +++++++--- cmd/ate-setup/internal/steps/publish_test.go | 10 +-- cmd/dataplane/envoy/README.md | 4 +- docs/dev/releasing.md | 12 ++-- docs/examples/substrate-install.yaml | 5 +- hack/release/draft-release.sh | 17 ++++- 19 files changed, 219 insertions(+), 100 deletions(-) diff --git a/.github/scripts/release/ghcr-login.sh b/.github/scripts/release/ghcr-login.sh index 9d6dadbbb8..61d9e7ada6 100755 --- a/.github/scripts/release/ghcr-login.sh +++ b/.github/scripts/release/ghcr-login.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Logs docker in to ghcr.io as GITHUB_ACTOR with the token in TOKEN. ko and -# ate-setup's registry checks read the credentials docker login stores. +# Logs docker in to ghcr.io as GITHUB_ACTOR with the token in TOKEN. ko, buildx, +# and ate-setup's registry checks read the credentials docker login stores. # # GITHUB_ACTOR= TOKEN= .github/scripts/release/ghcr-login.sh # diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b1b622c544..c14d6448e0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -80,9 +80,15 @@ jobs: # These jobs can write packages or releases, so they restore no cache # that another workflow could have written. cache: false - # Every release image is built with ko, which cross-compiles Go for arm64 - # without emulation, so the job needs neither QEMU nor buildx. ko and - # ate-setup's registry checks read the credentials docker login stores. + # ko cross-compiles Go for arm64, and the envoy-dataplane Dockerfile + # cross-compiles its Rust module, so the job needs no QEMU. buildx builds + # envoy-dataplane for both platforms. ko, buildx, and ate-setup's registry + # checks read the credentials docker login stores. BuildKit is pinned by + # digest because it builds a published image with the ghcr.io login. + - name: Set up buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + with: + driver-opts: image=moby/buildkit:v0.33.1@sha256:cec9f139f45e93c5c69c60f8b07cfad9f43f4ef6b6a6cd917527fea5ff2e3dea - name: Log in to ghcr.io env: TOKEN: ${{ github.token }} @@ -129,6 +135,17 @@ jobs: # These jobs can write packages or releases, so they restore no cache # that another workflow could have written. cache: false + # The same BuildKit as the images job. Host networking and plain HTTP let + # buildx push to the registry service. + - name: Set up buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + with: + driver-opts: | + image=moby/buildkit:v0.33.1@sha256:cec9f139f45e93c5c69c60f8b07cfad9f43f4ef6b6a6cd917527fea5ff2e3dea + network=host + buildkitd-config-inline: | + [registry."localhost:5000"] + http = true - name: Publish images run: go run ./cmd/ate-setup publish release-images draft: diff --git a/cmd/ate-setup/README.md b/cmd/ate-setup/README.md index dafb7671ad..17e1d8c248 100644 --- a/cmd/ate-setup/README.md +++ b/cmd/ate-setup/README.md @@ -21,13 +21,15 @@ tag they share: ``` ate-setup deploy ate-system \ --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=IDS \ --image-repo registry.example.com/substrate \ --image-tag vX.Y.Z ``` The registry has to hold every image the install deploys. The images a release -publishes to `ghcr.io/agent-substrate/substrate` are not enough on their own; -see [`docs/dev/releasing.md`](../../docs/dev/releasing.md#release-images). An +publishes to `ghcr.io/agent-substrate/substrate` leave out the demos and the +benchmarking fakes; see +[`docs/dev/releasing.md`](../../docs/dev/releasing.md#release-images). An install that needs an image the registry lacks fails naming it. Nothing is built and no registry is pushed to; the manifests still come from diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index a092188c92..267b790648 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -69,13 +69,15 @@ unchanged. ``` ate-setup deploy ate-system \ --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=IDS \ --image-repo REPO \ --image-tag TAG ``` installs images already published to `REPO` instead, and never invokes `ko`. The manifests still come from the checkout, so this needs one; what it removes -is the build, the Go toolchain, and write access to a registry. +is the build and write access to a registry. Releases ship no `ate-setup` +binary, so running it from the checkout still needs Go. `REPO` has to hold every image the install deploys, all under `TAG`, including `envoy-dataplane`, which a build from source builds from a Dockerfile with @@ -83,7 +85,7 @@ is the build, the Go toolchain, and write access to a registry. needs an image `REPO` lacks fails naming it. The images a release publishes to `ghcr.io/agent-substrate/substrate`, with -`publish release-images`, are not a complete install; see +`publish release-images`, leave out the demos and the benchmarking fakes; see [`docs/dev/releasing.md`](../../docs/dev/releasing.md#release-images). When an image releases do not publish is missing from `REPO`, the error says so. diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 9c53e93872..725594261f 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -146,8 +146,9 @@ to publish the new image under the same repository and tag as the rest, and until it does, an install has to build it or take it from a registry that holds it. -Releases publish only `images.ReleaseComponents`, an allowlist: a component is -published only once it is added to it, so a new entry in `images.Components` +Releases publish `images.ReleaseComponents` and +`images.ReleaseDockerfileImages`, both allowlists: an image is published only +once it is added to one, so a new entry in `images.Components` stays out of releases until someone adds it on purpose. See [`docs/dev/releasing.md`](../../docs/dev/releasing.md#release-images) for what releases cover. Under `--image-repo`, a component that is not on the allowlist @@ -258,8 +259,9 @@ through `ListActors`. The tolerances are preserved: no `ate-api-server`, or an a that cannot be reached, skips cleanup rather than failing, because `delete all` runs this for every demo. -**`docker buildx imagetools inspect` output is parsed as JSON**, not piped -through `jq -r '.manifest.digest'`. Build output still goes to stderr so the +**The pushed digest comes from `docker buildx build --metadata-file`**, not +from piping `docker buildx imagetools inspect` of the tag through +`jq -r '.manifest.digest'`. Build output still goes to stderr so the stdout/stderr split matches under CI log capture. ## Errors and diagnostics diff --git a/cmd/ate-setup/internal/cmd/publish.go b/cmd/ate-setup/internal/cmd/publish.go index 5294157ef2..02df1ef07f 100644 --- a/cmd/ate-setup/internal/cmd/publish.go +++ b/cmd/ate-setup/internal/cmd/publish.go @@ -52,15 +52,15 @@ worker ref.`, var publishReleaseImagesCmd = &cobra.Command{ Use: "release-images", Short: "Build and push the release images, tagged with the release version", - Long: `Build and push the images a release publishes (images.ReleaseComponents) to -KO_DOCKER_REPO, all tagged with the build version, and print their pushed -references. The version comes from VERSION, else git describe, and has to be a -release tag (vMAJOR.MINOR.PATCH[-PRERELEASE]). + Long: `Build and push the images a release publishes (images.ReleaseComponents, and +images.ReleaseDockerfileImages, which need docker buildx) to KO_DOCKER_REPO, all +tagged with the build version, and print their pushed references. The version +comes from VERSION, else git describe, and has to be a release tag +(vMAJOR.MINOR.PATCH[-PRERELEASE]). VERSION=TAG ate-setup publish release-images --ko-docker-repo REPO -The release images are not a complete install; docs/dev/releasing.md lists -what they cover. +docs/dev/releasing.md lists what the release images cover. If REPO already holds every image for the tag on every platform, nothing is built unless --overwrite is set. After pushing, every image is checked for diff --git a/cmd/ate-setup/internal/images/dockerfile.go b/cmd/ate-setup/internal/images/dockerfile.go index 78211a3c3c..a55df1c773 100644 --- a/cmd/ate-setup/internal/images/dockerfile.go +++ b/cmd/ate-setup/internal/images/dockerfile.go @@ -15,7 +15,6 @@ package images import ( - "bytes" "context" "encoding/json" "errors" @@ -69,13 +68,29 @@ func dockerfilePlatforms(rootDir, koDefaultPlatforms string) (string, error) { } // dockerBuildArgs returns the docker arguments that build contextPath for -// platforms and push it as tag. extraFlags go before the context. -func dockerBuildArgs(platforms string, extraFlags []string, tag, contextPath string) []string { - args := []string{"buildx", "build", "--platform=" + platforms, "--push"} +// platforms, push it as tag, and write the build metadata to metadataFile. +// extraFlags go before the context. +func dockerBuildArgs(platforms string, extraFlags []string, tag, metadataFile, contextPath string) []string { + args := []string{"buildx", "build", "--platform=" + platforms, "--push", "--metadata-file", metadataFile} args = append(args, extraFlags...) return append(args, "-t", tag, contextPath) } +// pushedDigest returns the digest buildx pushed, from the file written by +// --metadata-file. +func pushedDigest(metadata []byte) (string, error) { + var parsed struct { + Digest string `json:"containerimage.digest"` + } + if err := json.Unmarshal(metadata, &parsed); err != nil { + return "", fmt.Errorf("while parsing the buildx metadata: %w", err) + } + if parsed.Digest == "" { + return "", errors.New("the buildx metadata has no containerimage.digest") + } + return parsed.Digest, nil +} + // BuildDockerfileImage builds a Dockerfile-based image from contextPath, pushes // it to dockerRepo/, and returns the digest-pinned reference. // extraFlags are passed to docker buildx build, e.g. --cache-from/--cache-to. @@ -84,15 +99,30 @@ func dockerBuildArgs(platforms string, extraFlags []string, tag, contextPath str // push to; the returned reference always uses the digest, so a stale tag can // never be resolved by accident. func BuildDockerfileImage(ctx context.Context, rootDir, dockerRepo, imageName, contextPath, koDefaultPlatforms string, extraFlags []string) (string, error) { + return PublishDockerfileImage(ctx, rootDir, dockerRepo, imageName, fmt.Sprintf("build-%d", time.Now().Unix()), contextPath, koDefaultPlatforms, extraFlags) +} + +// PublishDockerfileImage is BuildDockerfileImage with the tag chosen by the +// caller, for a release that publishes every image under one tag. +func PublishDockerfileImage(ctx context.Context, rootDir, dockerRepo, imageName, tag, contextPath, koDefaultPlatforms string, extraFlags []string) (string, error) { repo := strings.TrimSuffix(dockerRepo, "/") + "/" + imageName - stageTag := fmt.Sprintf("%s:build-%d", repo, time.Now().Unix()) + stageTag := repo + ":" + tag platforms, err := dockerfilePlatforms(rootDir, koDefaultPlatforms) if err != nil { return "", err } + metadata, err := os.CreateTemp("", "buildx-metadata-*.json") + if err != nil { + return "", err + } + if err := metadata.Close(); err != nil { + return "", err + } + defer os.Remove(metadata.Name()) + build := exec.CommandContext(ctx, "docker", - dockerBuildArgs(platforms, extraFlags, stageTag, contextPath)...) + dockerBuildArgs(platforms, extraFlags, stageTag, metadata.Name(), contextPath)...) build.Dir = rootDir // The shell version sent build output to stderr so it could capture the // image reference on stdout; keeping that split makes the two behave the @@ -103,26 +133,13 @@ func BuildDockerfileImage(ctx context.Context, rootDir, dockerRepo, imageName, c return "", fmt.Errorf("while building the %s image: %w", imageName, err) } - inspect := exec.CommandContext(ctx, "docker", "buildx", "imagetools", "inspect", - stageTag, "--format", "{{json .}}") - inspect.Dir = rootDir - inspect.Stderr = os.Stderr - var out bytes.Buffer - inspect.Stdout = &out - if err := inspect.Run(); err != nil { - return "", fmt.Errorf("while inspecting %s: %w", stageTag, err) - } - - var inspected struct { - Manifest struct { - Digest string `json:"digest"` - } `json:"manifest"` - } - if err := json.Unmarshal(out.Bytes(), &inspected); err != nil { - return "", fmt.Errorf("while parsing the image manifest of %s: %w", stageTag, err) + raw, err := os.ReadFile(metadata.Name()) + if err != nil { + return "", err } - if inspected.Manifest.Digest == "" { - return "", fmt.Errorf("failed to resolve the image digest from %s", stageTag) + digest, err := pushedDigest(raw) + if err != nil { + return "", fmt.Errorf("%s: %w", stageTag, err) } - return repo + "@" + inspected.Manifest.Digest, nil + return repo + "@" + digest, nil } diff --git a/cmd/ate-setup/internal/images/dockerfile_test.go b/cmd/ate-setup/internal/images/dockerfile_test.go index 97fea6f08f..1bf04315c3 100644 --- a/cmd/ate-setup/internal/images/dockerfile_test.go +++ b/cmd/ate-setup/internal/images/dockerfile_test.go @@ -23,18 +23,48 @@ import ( ) func TestDockerBuildArgs(t *testing.T) { - got := dockerBuildArgs("linux/amd64", []string{"--cache-from", "type=gha"}, "repo/img:build-1", "ctx") - want := []string{"buildx", "build", "--platform=linux/amd64", "--push", "--cache-from", "type=gha", "-t", "repo/img:build-1", "ctx"} + got := dockerBuildArgs("linux/amd64", []string{"--cache-from", "type=gha"}, "repo/img:build-1", "meta.json", "ctx") + want := []string{"buildx", "build", "--platform=linux/amd64", "--push", "--metadata-file", "meta.json", "--cache-from", "type=gha", "-t", "repo/img:build-1", "ctx"} if !slices.Equal(got, want) { t.Errorf("dockerBuildArgs() = %q, want %q", got, want) } - got = dockerBuildArgs("linux/amd64", nil, "repo/img:build-1", "ctx") - want = []string{"buildx", "build", "--platform=linux/amd64", "--push", "-t", "repo/img:build-1", "ctx"} + got = dockerBuildArgs("linux/amd64", nil, "repo/img:build-1", "meta.json", "ctx") + want = []string{"buildx", "build", "--platform=linux/amd64", "--push", "--metadata-file", "meta.json", "-t", "repo/img:build-1", "ctx"} if !slices.Equal(got, want) { t.Errorf("dockerBuildArgs() without flags = %q, want %q", got, want) } } +// The digest comes from what buildx pushed, so a tag that names another push +// cannot stand in for it. +func TestPushedDigest(t *testing.T) { + const digest = "sha256:bab239626dc054856e882a2ef022dfa845834113d213697e1dccd27dfbfac1a5" + for _, tc := range []struct { + name string + metadata string + want string + wantErr bool + }{ + { + name: "multi-platform push", + metadata: `{"buildx.build.ref":"ate-release/ate-release0/x","containerimage.digest":"` + digest + `","image.name":"localhost:5000/substrate/envoy-dataplane:v1"}`, + want: digest, + }, + {name: "no digest", metadata: `{"buildx.build.ref":"ate-release/ate-release0/x"}`, wantErr: true}, + {name: "not JSON", metadata: ``, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := pushedDigest([]byte(tc.metadata)) + if (err != nil) != tc.wantErr { + t.Fatalf("pushedDigest() error = %v, want error %v", err, tc.wantErr) + } + if got != tc.want { + t.Errorf("pushedDigest() = %q, want %q", got, tc.want) + } + }) + } +} + // ko builds for .ko.yaml's defaultPlatforms unless KO_DEFAULTPLATFORMS says // otherwise, and a Dockerfile image installed beside the ko images has to // cover the same platforms or it cannot run on some of their nodes. diff --git a/cmd/ate-setup/internal/images/images.go b/cmd/ate-setup/internal/images/images.go index 56944d9ad0..99151a405d 100644 --- a/cmd/ate-setup/internal/images/images.go +++ b/cmd/ate-setup/internal/images/images.go @@ -80,6 +80,25 @@ var released = []string{ // ReleaseComponents returns the Components a release publishes. func ReleaseComponents() []string { return slices.Clone(released) } +// DockerfileImage is an image built from a Dockerfile with docker buildx +// rather than with ko. +type DockerfileImage struct { + // Name is the image name under the repository. + Name string + // Context is the build context, relative to the repository root. + Context string +} + +// EnvoyDataplane is the image the envoy router and egress gateways run. +var EnvoyDataplane = DockerfileImage{Name: "envoy-dataplane", Context: "cmd/dataplane/envoy"} + +// releasedDockerfiles is the allowlist of DockerfileImages a release +// publishes, the counterpart of released. +var releasedDockerfiles = []DockerfileImage{EnvoyDataplane} + +// ReleaseDockerfileImages returns the DockerfileImages a release publishes. +func ReleaseDockerfileImages() []DockerfileImage { return slices.Clone(releasedDockerfiles) } + // IsReleased reports whether a release publishes pkg. func IsReleased(pkg string) bool { return slices.Contains(released, pkg) } diff --git a/cmd/ate-setup/internal/images/release.go b/cmd/ate-setup/internal/images/release.go index 72fc7f9d95..75d30f1213 100644 --- a/cmd/ate-setup/internal/images/release.go +++ b/cmd/ate-setup/internal/images/release.go @@ -42,6 +42,11 @@ const SourceURL = "https://github.com/agent-substrate/substrate" // from. const RevisionLabel = "org.opencontainers.image.revision" +// VersionLabel is the OCI image label naming the release an image belongs to. +// It replaces any value inherited from a base image, such as the Ubuntu +// release envoyproxy/envoy carries. +const VersionLabel = "org.opencontainers.image.version" + // releaseTag matches a release tag, vMAJOR.MINOR.PATCH, or a release candidate, // vMAJOR.MINOR.PATCH-PRERELEASE. hack/release/draft-release.sh accepts the same. var releaseTag = regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$`) diff --git a/cmd/ate-setup/internal/steps/env.go b/cmd/ate-setup/internal/steps/env.go index 6af68a5c06..26552ca1e1 100644 --- a/cmd/ate-setup/internal/steps/env.go +++ b/cmd/ate-setup/internal/steps/env.go @@ -172,16 +172,16 @@ func (e *Env) koRunner() (*ko.Runner, error) { // KO_DOCKER_REPO. A pre-built install builds nothing: --image-repo installs // images someone else published, and needs neither docker nor a registry to // push to, so the image is pinned from REPO/name:TAG like every ko image. -// That only works if someone published it there: `ate-setup publish -// release-images` does not, so an install from release images fails here. -func (e *Env) dockerfileImage(ctx context.Context, name, contextDir string) (string, error) { - if ref, ok := e.dockerfileImages[name]; ok { +// That only works if the repository holds it and can be read: releases +// publish it, but older releases and other mirrors may not. +func (e *Env) dockerfileImage(ctx context.Context, img images.DockerfileImage) (string, error) { + if ref, ok := e.dockerfileImages[img.Name]; ok { return ref, nil } var ref string var err error if !e.Cfg.Images.IsPrebuilt() { - ref, err = images.BuildDockerfileImage(ctx, e.Cfg.Root, e.Cfg.KODockerRepo, name, e.Cfg.Path(contextDir), e.Cfg.KODefaultPlatforms, e.Cfg.DockerBuildFlags) + ref, err = images.BuildDockerfileImage(ctx, e.Cfg.Root, e.Cfg.KODockerRepo, img.Name, e.Cfg.Path(img.Context), e.Cfg.KODefaultPlatforms, e.Cfg.DockerBuildFlags) } else { resolver, rErr := e.imageResolver() if rErr != nil { @@ -191,11 +191,11 @@ func (e *Env) dockerfileImage(ctx context.Context, name, contextDir string) (str if !ok { return "", fmt.Errorf("image resolver is %T, not a pre-built resolver", resolver) } - ref, err = prebuilt.Pin(ctx, name) + ref, err = prebuilt.Pin(ctx, img.Name) if err != nil { - return "", fmt.Errorf("%s is built from a Dockerfile, and releases do not publish it; "+ + return "", fmt.Errorf("%s is built from a Dockerfile, and the image repository has no readable copy; "+ "build it from source by dropping --image-repo, or choose --atenet-dataplane=agentgateway, "+ - "which does not use it: %w", name, err) + "which does not use it: %w", img.Name, err) } } if err != nil { @@ -204,7 +204,7 @@ func (e *Env) dockerfileImage(ctx context.Context, name, contextDir string) (str if e.dockerfileImages == nil { e.dockerfileImages = make(map[string]string) } - e.dockerfileImages[name] = ref + e.dockerfileImages[img.Name] = ref return ref, nil } diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 89d82ca4f3..aa15059888 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -26,6 +26,7 @@ import ( "time" "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" "go.ate.dev/substrate/cmd/ate-setup/internal/kustomize" "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) @@ -33,10 +34,6 @@ import ( const ( // installDir is the manifest root, relative to the repository root. installDir = "manifests/ate-install" - // Envoy dataplane image name - envoyDataplaneImage = "envoy-dataplane" - // Envoy dataplane Dockerfile path - envoyDataplaneDockefile = "cmd/dataplane/envoy" ) // cordonControlPlaneComponent is the kustomize component that pins each @@ -156,7 +153,7 @@ func (e *Env) renderSystemManifests(ctx context.Context) ([]byte, error) { if err != nil { return nil, err } - imageReference, err := e.dockerfileImage(ctx, envoyDataplaneImage, envoyDataplaneDockefile) + imageReference, err := e.dockerfileImage(ctx, images.EnvoyDataplane) if err != nil { return nil, err } @@ -187,7 +184,7 @@ func (e *Env) renderAtenetRouterManifest(ctx context.Context) ([]byte, error) { if err != nil { return nil, err } - imageReference, err := e.dockerfileImage(ctx, envoyDataplaneImage, envoyDataplaneDockefile) + imageReference, err := e.dockerfileImage(ctx, images.EnvoyDataplane) if err != nil { return nil, err } @@ -245,7 +242,7 @@ func (e *Env) renderAtenetEgressManifest(ctx context.Context, provider config.Cr return e.ResolveManifestBytes(ctx, raw) } - imageReference, err := e.dockerfileImage(ctx, envoyDataplaneImage, envoyDataplaneDockefile) + imageReference, err := e.dockerfileImage(ctx, images.EnvoyDataplane) if err != nil { return nil, err } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 814ca752e1..2fe3ae34e3 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -1300,8 +1300,7 @@ func TestApplyOtelEndpointOverride(t *testing.T) { // A pre-built install renders the envoy router, egress, and system manifests // without building anything: envoy-dataplane is pinned from the image // repository like every ko image, so neither docker nor KO_DOCKER_REPO is -// needed. Releases do not publish it, so this holds for a repository someone -// published it to. +// needed. func TestRenderAtenetEgressManifestPrebuilt(t *testing.T) { const digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222" src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} @@ -1353,9 +1352,8 @@ func TestRenderAtenetEgressManifestPrebuilt(t *testing.T) { } } -// An install from release images that needs envoy-dataplane fails with a -// message saying releases do not publish it and how to proceed, rather than a -// bare registry error. +// An install from an image repository without a readable envoy-dataplane fails +// with a message saying how to proceed, rather than a bare registry error. func TestDockerfileImagePrebuiltNotPublished(t *testing.T) { src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} e := &Env{ @@ -1365,12 +1363,12 @@ func TestDockerfileImagePrebuiltNotPublished(t *testing.T) { }), } - _, err := e.dockerfileImage(t.Context(), envoyDataplaneImage, envoyDataplaneDockefile) + _, err := e.dockerfileImage(t.Context(), images.EnvoyDataplane) if err == nil { t.Fatal("dockerfileImage() error = nil, want one") } for _, want := range []string{ - "envoy-dataplane is built from a Dockerfile, and releases do not publish it", + "envoy-dataplane is built from a Dockerfile, and the image repository has no readable copy", "--atenet-dataplane=agentgateway", "resolving example.com/substrate/envoy-dataplane:v1.2.3 to a digest: MANIFEST_UNKNOWN", } { diff --git a/cmd/ate-setup/internal/steps/publish.go b/cmd/ate-setup/internal/steps/publish.go index cf36277fae..f98326e733 100644 --- a/cmd/ate-setup/internal/steps/publish.go +++ b/cmd/ate-setup/internal/steps/publish.go @@ -168,22 +168,24 @@ type PublishReleaseOptions struct { } // releaseImageRefs returns the tagged reference of every image a release -// publishes to repo: one per images.ReleaseComponents entry. +// publishes to repo: one per images.ReleaseComponents and +// images.ReleaseDockerfileImages entry. func releaseImageRefs(repo, tag string) []string { repo = strings.TrimSuffix(repo, "/") - pkgs := images.ReleaseComponents() - refs := make([]string, 0, len(pkgs)) - for _, pkg := range pkgs { + var refs []string + for _, pkg := range images.ReleaseComponents() { refs = append(refs, repo+"/"+images.ImageName(pkg)+":"+tag) } + for _, img := range images.ReleaseDockerfileImages() { + refs = append(refs, repo+"/"+img.Name+":"+tag) + } return refs } // PublishReleaseImages builds and pushes every image a release publishes, the -// ko images in images.ReleaseComponents, all tagged with the build version, -// and writes their pushed references to w. The version has to be a release -// tag. Releases do not publish envoy-dataplane, the one Dockerfile-built -// image, so nothing here needs docker. +// ko images in images.ReleaseComponents and the images.ReleaseDockerfileImages +// built with docker buildx, all tagged with the build version, and writes +// their pushed references to w. The version has to be a release tag. // // If the registry already holds every image for the tag on every platform, // nothing is built unless opts.Overwrite is set, so rerunning a release never @@ -234,11 +236,29 @@ func (e *Env) PublishReleaseImages(ctx context.Context, w io.Writer, opts Publis labels := []string{ images.SourceLabel + "=" + images.SourceURL, images.RevisionLabel + "=" + revision, + images.VersionLabel + "=" + tag, + } + // The Dockerfile images are slower to build and likelier to fail, so they + // go first: a failure there publishes nothing under the tag. They ignore + // DOCKER_BUILD_FLAGS, so a release restores no build cache. + var dockerFlags []string + for _, label := range labels { + dockerFlags = append(dockerFlags, "--label="+label) + } + var refs []string + for _, img := range images.ReleaseDockerfileImages() { + ref, err := images.PublishDockerfileImage(ctx, e.Cfg.Root, e.Cfg.KODockerRepo, img.Name, tag, + e.Cfg.Path(img.Context), e.Cfg.KODefaultPlatforms, dockerFlags) + if err != nil { + return err + } + refs = append(refs, ref) } - refs, err := runner.BuildTagged(ctx, tag, labels, pkgs...) + koRefs, err := runner.BuildTagged(ctx, tag, labels, pkgs...) if err != nil { return err } + refs = append(refs, koRefs...) incomplete, err := images.CheckPublished(ctx, tagged, platforms) if err != nil { diff --git a/cmd/ate-setup/internal/steps/publish_test.go b/cmd/ate-setup/internal/steps/publish_test.go index 96a6c59fbe..686af7f105 100644 --- a/cmd/ate-setup/internal/steps/publish_test.go +++ b/cmd/ate-setup/internal/steps/publish_test.go @@ -120,16 +120,17 @@ func TestPublishReleaseImagesRejectsANonReleaseTag(t *testing.T) { } } -// The release covers every released component, each under the name and tag -// that `deploy --image-repo --image-tag` looks up, and nothing releases leave -// out. +// The release covers every released component and envoy-dataplane, each under +// the name and tag that `deploy --image-repo --image-tag` looks up, and nothing +// releases leave out. func TestReleaseImageRefs(t *testing.T) { refs := releaseImageRefs("example.com/substrate/", "v1") - if want := len(images.ReleaseComponents()); len(refs) != want { + if want := len(images.ReleaseComponents()) + len(images.ReleaseDockerfileImages()); len(refs) != want { t.Fatalf("releaseImageRefs() returned %d refs, want %d", len(refs), want) } for _, want := range []string{ "example.com/substrate/ateapi:v1", + "example.com/substrate/envoy-dataplane:v1", "example.com/substrate/kubernetes-secrets:v1", "example.com/substrate/objectstore-plugin:v1", } { @@ -139,7 +140,6 @@ func TestReleaseImageRefs(t *testing.T) { } for _, unwanted := range []string{ "example.com/substrate/counter:v1", - "example.com/substrate/envoy-dataplane:v1", } { if slices.Contains(refs, unwanted) { t.Errorf("releaseImageRefs() = %v, want no %s", refs, unwanted) diff --git a/cmd/dataplane/envoy/README.md b/cmd/dataplane/envoy/README.md index 7399619fae..29730d1043 100644 --- a/cmd/dataplane/envoy/README.md +++ b/cmd/dataplane/envoy/README.md @@ -21,9 +21,9 @@ cmd/dataplane/envoy/ During a build from source, `ate-setup` builds the image from this directory via `docker buildx`, pushes it to `$KO_DOCKER_REPO/envoy-dataplane`, and its resolved digest replaces the `${ENVOY_DATAPLANE_IMAGE}` placeholder in the router and egress manifests (`manifests/ate-install/atenet-router.yaml`, `manifests/ate-install/atenet-egress.yaml`). -A pre-built install (`ate-setup deploy --image-repo REPO --image-tag TAG`) builds nothing and pins `REPO/envoy-dataplane:TAG` instead, so `REPO` has to hold it. Releases do not publish it; see [`docs/dev/releasing.md`](../../../docs/dev/releasing.md#release-images). +A pre-built install (`ate-setup deploy --image-repo REPO --image-tag TAG`) builds nothing and pins `REPO/envoy-dataplane:TAG` instead, so `REPO` has to hold it. Releases publish it there with the other images; see [`docs/dev/releasing.md`](../../../docs/dev/releasing.md#release-images). -The image targets the same platforms as the ko images: `KO_DEFAULTPLATFORMS`, else `defaultPlatforms` in `.ko.yaml` (linux/amd64 and linux/arm64), else `linux/amd64`. Building another architecture compiles Rust under QEMU, which must be registered with binfmt (e.g. `docker run --privileged --rm tonistiigi/binfmt --install arm64`). +The image targets the same platforms as the ko images: `KO_DEFAULTPLATFORMS`, else `defaultPlatforms` in `.ko.yaml` (linux/amd64 and linux/arm64), else `linux/amd64`. The builder stage runs on the build machine's platform and cross-compiles the Rust module, so building another architecture needs no QEMU. To build the image locally without pushing it: diff --git a/docs/dev/releasing.md b/docs/dev/releasing.md index 029e17f744..dccf4b55d0 100644 --- a/docs/dev/releasing.md +++ b/docs/dev/releasing.md @@ -13,8 +13,8 @@ The ruleset limits tag pushes only. Anyone with write access can still publish i ## Cut a release 1. Tag the release commit and push the tag. The [`release`](../../.github/workflows/release.yaml) workflow does two things, in order: - 1. The `images` job runs `ate-setup publish release-images`, which builds every release image for `linux/amd64` and `linux/arm64`, pushes it to `ghcr.io/agent-substrate/substrate/:`, and then checks that every tag names the image just pushed, for both platforms. Every release image is built with `ko`, which cross-compiles for arm64 without emulation. - 2. Once the images are verified, the `draft` job builds the `kubectl-ate` binaries with `ate-setup publish release-binaries`, then runs [`hack/release/draft-release.sh`](../../hack/release/draft-release.sh), which creates a draft release, and attaches the binaries to it. The notes cover every change since the previous `vX.Y.Z` release, say where the release images are published and link to the list below, give the steps to check and install `kubectl-ate`, and end with a list of committers. A release candidate is marked as a prerelease. + 1. The `images` job runs `ate-setup publish release-images`, which builds every release image for `linux/amd64` and `linux/arm64`, pushes it to `ghcr.io/agent-substrate/substrate/:`, and then checks that every tag names the image just pushed, for both platforms. The ko images and `envoy-dataplane`, built from its Dockerfile with `docker buildx`, all cross-compile for arm64 without emulation. + 2. Once the images are verified, the `draft` job builds the `kubectl-ate` binaries with `ate-setup publish release-binaries`, then runs [`hack/release/draft-release.sh`](../../hack/release/draft-release.sh), which creates a draft release, and attaches the binaries to it. The notes cover every change since the previous `vX.Y.Z` release, say where the release images are published and link to the list below, give the command that installs them, give the steps to check and install `kubectl-ate`, and end with a list of committers. A release candidate is marked as a prerelease. If the `images` job or the binary build fails, no draft is created. If attaching the binaries fails, the draft exists without them. Either way, fix the cause and run the workflow again from the Actions tab with the tag as input. 2. Edit the draft: @@ -37,11 +37,11 @@ To preview the notes before tagging, run the dry run with the tag you plan to pu ## Release images -A release publishes `ate-idp-server`, `ateapi`, `atecontroller`, `atelet`, `atenet`, `ateworker-gvisor`, `ateworker-microvm`, `kubernetes-secrets` (the Kubernetes Secrets credential provider), `objectstore-plugin`, and `podcertcontroller`, as `ghcr.io/agent-substrate/substrate/:` (`images.ReleaseComponents` in [`cmd/ate-setup/internal/images`](../../cmd/ate-setup/internal/images/images.go)). The object storage plugin ships with releases until it moves out of this repository. +A release publishes `ate-idp-server`, `ateapi`, `atecontroller`, `atelet`, `atenet`, `ateworker-gvisor`, `ateworker-microvm`, `kubernetes-secrets` (the Kubernetes Secrets credential provider), `objectstore-plugin`, and `podcertcontroller` (`images.ReleaseComponents` in [`cmd/ate-setup/internal/images`](../../cmd/ate-setup/internal/images/images.go)), plus `envoy-dataplane` from [`cmd/dataplane/envoy`](../../cmd/dataplane/envoy), as `ghcr.io/agent-substrate/substrate/:`. The object storage plugin ships with releases until it moves out of this repository. -The list is an allowlist, `released` in `images.go`. A new component is not published until a pull request adds it there, so publishing an image is always an explicit, reviewed change. +Both lists are allowlists in `images.go`: `released` for the ko images and `releasedDockerfiles` for the images built from a Dockerfile. A new image is not published until a pull request adds it there, so publishing an image is always an explicit, reviewed change. -It leaves out `envoy-dataplane`, the demos, and the benchmarking fakes. The release images are therefore not a complete install of the default configuration, and the notes give no install command: an install has to take the images releases leave out from another registry, or build them from source. `ate-setup --image-repo` looks those images up like any other; if one is missing, the error says releases do not publish it. +It leaves out the demos and the benchmarking fakes, which an install deploys only when asked to. An install that needs them has to take them from another registry, or build them from source. `ate-setup --image-repo` looks those images up like any other; if one is missing, the error says releases do not publish it. Tags that begin `vX.Y.Z` start the workflow. `ate-setup` refuses any version that is not `vX.Y.Z` or `vX.Y.Z-PRERELEASE` before building, so a tag like `v1.2.3foo` publishes nothing. @@ -60,7 +60,7 @@ VERSION=v0.3.0 KO_DEFAULTPLATFORMS=linux/amd64,linux/arm64 \ go run ./cmd/ate-setup publish release-images --ko-docker-repo ``` -The images are built from the work tree, so it must hold the tag and nothing else. The build needs only Go and `ko`, which `ate-setup` runs from `hack/tools/ko`. +The images are built from the work tree, so it must hold the tag and nothing else. The build needs Go, `ko` (which `ate-setup` runs from `hack/tools/ko`), and `docker buildx` for `envoy-dataplane`. A multi-platform push needs a builder that supports it, such as one made with `docker buildx create --driver docker-container --use`; the default `docker` driver supports it only with the containerd image store. ## Release binaries diff --git a/docs/examples/substrate-install.yaml b/docs/examples/substrate-install.yaml index e1077f6563..a2ca9618da 100644 --- a/docs/examples/substrate-install.yaml +++ b/docs/examples/substrate-install.yaml @@ -42,9 +42,8 @@ atenet: # Prebuilt images. Both halves are required together: a tag with no repository # would be dropped, leaving a build from source that looks like the release the -# tag names. The repository has to hold every image the install deploys; the -# release images in ghcr.io/agent-substrate/substrate are not a complete set -# (see docs/dev/releasing.md). +# tag names. The repository has to hold every image the install deploys; see +# docs/dev/releasing.md for what releases publish. images: repo: registry.example.com/substrate tag: v0.4.1 diff --git a/hack/release/draft-release.sh b/hack/release/draft-release.sh index 7980da2dd2..d598bc9fea 100755 --- a/hack/release/draft-release.sh +++ b/hack/release/draft-release.sh @@ -147,14 +147,25 @@ if ! notes="$(api "generating notes" "repos/${REPO}/releases/generate-notes" \ fi # Only the upstream release workflow publishes images (see -# .github/workflows/release.yaml), so only its notes say where they are. The -# notes give no install command because the release images are not a complete -# install; docs/dev/releasing.md lists what they cover. +# .github/workflows/release.yaml), so only its notes say where they are. +# docs/dev/releasing.md lists what they cover. if [[ "${REPO}" == "${UPSTREAM_REPO}" ]]; then notes+=$'\n\n## Images\n\n' notes+="Container images for this release are published as \`${RELEASE_IMAGE_REPO}/:${tag}\`" notes+=" for \`linux/amd64\` and \`linux/arm64\`. The images a release publishes are listed in" notes+=" [Release images](https://github.com/${REPO}/blob/${tag}/docs/dev/releasing.md#release-images)." + notes+=$'\n\n## Install\n\n' + notes+="From a checkout of \`${tag}\`, with Go, install the release images into the current kubectl context." + notes+=" Add \`--kind\` for a kind cluster." + notes+=$'\n\n```sh\n' + notes+="git clone --depth 1 --branch ${tag} https://github.com/${REPO}.git"$'\n' + notes+=$'cd substrate\n' + notes+=$'go run ./cmd/ate-setup deploy ate-system \\\n' + notes+=" --image-repo ${RELEASE_IMAGE_REPO} --image-tag ${tag} \\"$'\n' + notes+=$' --credential-provider=\'{"name":"k8s.io"}\' \\\n' + notes+=$' --ingress-allowed-spiffe-ids=\n' + notes+='```' + notes+=$'\n\n'"See [Installing pre-built images](https://github.com/${REPO}/blob/${tag}/cmd/ate-setup/commands.md#installing-pre-built-images) for the flags." # The workflow attaches these after creating the draft; see the # "Attach kubectl-ate" step in .github/workflows/release.yaml. notes+=$'\n\n## kubectl-ate\n\n' From 7e6476cbda376f3ee19d0ee507d85ba6e5cd7322 Mon Sep 17 00:00:00 2001 From: Chenyi Wang Date: Tue, 27 Oct 2026 22:03:00 +0000 Subject: [PATCH 19/53] atelet, ateworker: finish a checkpoint retry after atelet restarts (#3218) Follow-up to #2836. A Checkpoint retry still failed with `runsc checkpoint: exit status 128` when atelet died before the snapshot committed: while staging the local snapshot or after publishing it (both reported by @eren23), or while the worker was still capturing the workload, which the worker then canceled. - atelet stages the snapshot files as hard links, so checkpoint-state keeps them until the commit, and a retry replaces an uncommitted local snapshot. - The workers finish CheckpointWorkload under the caller's deadline but not its cancellation. TestRetriedCheckpointAfterUncommittedStaging and TestCheckpointContext cover the change, and the TestCheckpointBoundaryAudit fixture from #2836 passes all 15 cases, up from 9. The "pause move fails" row of TestFailedCheckpointKeepsLocalSnapshot is removed, since a leftover local snapshot directory is now replaced. AI use: AI-assisted. I reviewed and tested it. - [x] Tests pass --------- Signed-off-by: Chenyi Wang --- cmd/atelet/internal/ateletpath/ateletpath.go | 4 +- cmd/atelet/lifecycle_test.go | 124 ++++++++++++++++--- cmd/atelet/main.go | 13 +- cmd/ateworker-gvisor/main.go | 2 +- cmd/ateworker-microvm/checkpoint.go | 2 +- internal/ateworker/checkpoint.go | 13 ++ internal/ateworker/checkpoint_test.go | 37 ++++++ 7 files changed, 171 insertions(+), 24 deletions(-) diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index 67bab746c1..f28ff09c45 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -75,8 +75,8 @@ func ActorPath(actorUID string) string { } // ActorSnapshotPath is the actor's snapshot staging directory, which may be on -// a different filesystem from ActorPath. Checkpoints rename files between its -// subdirectories, so they must stay on one filesystem. +// a different filesystem from ActorPath. Checkpoints link and rename files +// between its subdirectories, so they must stay on one filesystem. func ActorSnapshotPath(actorUID string) string { return filepath.Join( nodepath.SnapshotsDir, diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 1d96035c65..72eccf43d5 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -74,7 +74,8 @@ func useTempNodeDirs(t *testing.T) { // the request, never derived from the actor UID. type fakeWorker struct { ateworkerpb.UnimplementedWorkerServer - // mu guards the record maps: concurrent RPCs record at the same time. + // mu guards the record maps and stoppedByResp: concurrent RPCs record at + // the same time. mu sync.Mutex // snapshotFiles are written at checkpoint and reported back to atelet as // the exact set the snapshot consists of. @@ -104,10 +105,34 @@ type fakeWorker struct { // onRestore, if non-nil, runs inside RestoreWorkload after it recorded // the request, before it fails with restoreErr or succeeds. onRestore func() + // stopsWorkload, if set, makes a checkpoint stop the workload, as a real + // worker's does. Until a Run or Restore starts it again, CheckpointWorkload + // answers with the stopping checkpoint's response while the checkpoint dir + // still holds its files, and fails otherwise. + stopsWorkload bool + // stoppedByResp is the response of the checkpoint that stopped the + // workload, nil while the workload runs. Guarded by mu. + stoppedByResp *ateworkerpb.CheckpointWorkloadResponse // terminations counts TerminateWorkload calls. terminations atomic.Int32 } +// setStoppedBy records the response of the checkpoint that stopped the +// workload, or nil once the workload runs again. +func (f *fakeWorker) setStoppedBy(resp *ateworkerpb.CheckpointWorkloadResponse) { + f.mu.Lock() + defer f.mu.Unlock() + f.stoppedByResp = resp +} + +// stoppedBy returns the response of the checkpoint that stopped the +// workload, or nil while the workload runs. +func (f *fakeWorker) stoppedBy() *ateworkerpb.CheckpointWorkloadResponse { + f.mu.Lock() + defer f.mu.Unlock() + return f.stoppedByResp +} + func (f *fakeWorker) recordFidelity(rpc string, fidelity ateworkerpb.SnapshotFidelity) { f.mu.Lock() defer f.mu.Unlock() @@ -128,6 +153,7 @@ func (f *fakeWorker) recordActorDirs(rpc string, actorDirs *ateworkerpb.ActorDir func (f *fakeWorker) RunWorkload(_ context.Context, req *ateworkerpb.RunWorkloadRequest) (*ateworkerpb.RunWorkloadResponse, error) { f.recordActorDirs("RunWorkload", req.GetActorDirs()) + f.setStoppedBy(nil) return &ateworkerpb.RunWorkloadResponse{WorkerEpoch: f.workerEpoch}, nil } @@ -135,6 +161,14 @@ func (f *fakeWorker) CheckpointWorkload(ctx context.Context, req *ateworkerpb.Ch f.recordActorDirs("CheckpointWorkload", req.GetActorDirs()) f.recordFidelity("CheckpointWorkload", req.GetFidelity()) dir := req.GetActorDirs().GetCheckpointDir() + if resp := f.stoppedBy(); resp != nil { + for _, name := range resp.GetSnapshotFiles() { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + return nil, status.Errorf(codes.Internal, "no workload to checkpoint: %v", err) + } + } + return resp, nil + } names := make([]string, 0, len(f.snapshotFiles)) for name, body := range f.snapshotFiles { if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { @@ -145,10 +179,14 @@ func (f *fakeWorker) CheckpointWorkload(ctx context.Context, req *ateworkerpb.Ch if f.onCheckpoint != nil { f.onCheckpoint() } + resp := &ateworkerpb.CheckpointWorkloadResponse{SnapshotFiles: names} + if f.stopsWorkload { + f.setStoppedBy(resp) + } if err := ctx.Err(); err != nil { return nil, err } - return &ateworkerpb.CheckpointWorkloadResponse{SnapshotFiles: names}, nil + return resp, nil } func (f *fakeWorker) RestoreWorkload(_ context.Context, req *ateworkerpb.RestoreWorkloadRequest) (*ateworkerpb.RestoreWorkloadResponse, error) { @@ -161,6 +199,7 @@ func (f *fakeWorker) RestoreWorkload(_ context.Context, req *ateworkerpb.Restore if f.restoreErr != nil { return nil, f.restoreErr } + f.setStoppedBy(nil) dir := req.GetActorDirs().GetRestoreDir() f.restored = map[string]string{} for name := range f.snapshotFiles { @@ -1479,9 +1518,10 @@ func (a *pluginActor) snapshot(name string, generation int32) *ateletpb.Snapshot return lifecycleSnapshot(pluginActorUID, name, generation, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) } -func (a *pluginActor) checkpoint(t *testing.T, snap *ateletpb.Snapshot, mode ateletpb.UploadMode) (*ateletpb.CheckpointResponse, error) { - t.Helper() - return a.s.Checkpoint(t.Context(), &ateletpb.CheckpointRequest{ +// checkpointRequest is the CheckpointRequest the control plane sends to take +// snap under mode. +func (a *pluginActor) checkpointRequest(snap *ateletpb.Snapshot, mode ateletpb.UploadMode) *ateletpb.CheckpointRequest { + return &ateletpb.CheckpointRequest{ Atespace: "ate-demo", ActorName: "counter", ActorUid: pluginActorUID, @@ -1491,7 +1531,12 @@ func (a *pluginActor) checkpoint(t *testing.T, snap *ateletpb.Snapshot, mode ate Spec: a.spec, Snapshot: snap, UploadConfig: &ateletpb.UploadConfig{Mode: mode}, - }) + } +} + +func (a *pluginActor) checkpoint(t *testing.T, snap *ateletpb.Snapshot, mode ateletpb.UploadMode) (*ateletpb.CheckpointResponse, error) { + t.Helper() + return a.s.Checkpoint(t.Context(), a.checkpointRequest(snap, mode)) } // restoreLocal resumes the actor from its local snapshot named name, taken at @@ -1784,6 +1829,62 @@ func TestRetriedCheckpointAnswersFromCommittedSnapshot(t *testing.T) { } } +// TestRetriedCheckpointAfterUncommittedStaging retries a deferred Checkpoint +// whose first attempt died after the worker stopped the workload and atelet +// began staging the local snapshot, but before the snapshot committed. The +// checkpoint dir still holds the worker's files, so the worker answers the +// retry from them, and the retry commits the snapshot. +func TestRetriedCheckpointAfterUncommittedStaging(t *testing.T) { + for _, tc := range []struct { + name string + // notWritten names snapshot files the worker did not write. Staging + // stops at the first of them, as it would if atelet died there. + notWritten []string + }{ + {name: "staging interrupted", notWritten: []string{"pages.img"}}, + {name: "local snapshot published"}, + } { + t.Run(tc.name, func(t *testing.T) { + a := runPluginActor(t) + a.worker.stopsWorkload = true + var captures atomic.Int32 + a.worker.onCheckpoint = func() { captures.Add(1) } + snap := a.snapshot("snap-1", 1) + req := a.checkpointRequest(snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + + // The first attempt gets as far as staging the local snapshot. + wresp, err := a.worker.CheckpointWorkload(t.Context(), &ateworkerpb.CheckpointWorkloadRequest{ + ActorDirs: ateletpath.ActorDirs(pluginActorUID), + Fidelity: toWorkerFidelity(snap.GetFidelity()), + }) + if err != nil { + t.Fatalf("CheckpointWorkload: %v", err) + } + rec, err := readSandboxRecord(pluginActorUID) + if err != nil { + t.Fatal(err) + } + rec.SnapshotFiles = slices.Concat(wresp.GetSnapshotFiles(), tc.notWritten) + err = a.s.moveLocalCheckpoint(t.Context(), req, rec) + if wantErr := len(tc.notWritten) > 0; (err != nil) != wantErr { + t.Fatalf("moveLocalCheckpoint = %v, want error %t", err, wantErr) + } + + resp, err := a.s.Checkpoint(t.Context(), req) + if err != nil { + t.Fatalf("retried Checkpoint: %v", err) + } + if got := resp.GetState().GetLatestLocal().GetSnapshot(); !proto.Equal(got, snap) { + t.Errorf("latest_local = %v, want %v", got, snap) + } + if n := captures.Load(); n != 1 { + t.Errorf("worker captured the workload %d times, want once", n) + } + a.restoreLocal(t, "snap-1", 1) + }) + } +} + // TestTerminateWaitsForInFlightRPC checks that Terminate waits on the // per-actor lock while Checkpoint or PromoteSnapshot is in flight, then cleans // up the actor's tracked state and local files once that RPC returns. @@ -1912,17 +2013,6 @@ func TestFailedCheckpointKeepsLocalSnapshot(t *testing.T) { a.store.putErr = errors.New("object store unavailable") }, }, - { - name: "pause move fails", - snap: "pause-snap-2", - mode: ateletpb.UploadMode_UPLOAD_MODE_DEFERRED, - fail: func(t *testing.T, a *pluginActor) { - // A file where the new snapshot's directory belongs fails the move. - if err := os.WriteFile(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-2"), nil, 0o600); err != nil { - t.Fatal(err) - } - }, - }, { name: "missing snapshot file", snap: "pause-snap-2", diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 2f708a8778..4682bf0f00 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -1302,7 +1302,9 @@ func (s *Atelet) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Checkpoi // Stage the files worker reported and the snapshot manifest inside // checkpointDir before renaming the directory into local-checkpoint/, so a // failure leaves nothing under local-checkpoint/ and ResetActorDirs cleans - // checkpointDir up. + // checkpointDir up. Staging links the files, so checkpointDir still holds + // them for a retried CheckpointWorkload if atelet dies before the snapshot + // commits. stageDir := filepath.Join(checkpointDir, ".staged") if err := root.RemoveAll(stageDir); err != nil { return fmt.Errorf("while clearing local checkpoint staging directory: %w", err) @@ -1326,8 +1328,8 @@ func (s *Atelet) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Checkpoi if err := root.MkdirAll(filepath.Dir(dst), 0o700); err != nil { return fmt.Errorf("while creating parent directory for %s: %w", dst, err) } - if err := root.Rename(src, dst); err != nil { - return fmt.Errorf("failed to move %s to %s: %w", src, dst, err) + if err := root.Link(src, dst); err != nil { + return fmt.Errorf("failed to link %s to %s: %w", src, dst, err) } } @@ -1346,6 +1348,11 @@ func (s *Atelet) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Checkpoi if err := root.MkdirAll(filepath.Dir(localDir), 0o700); err != nil { return fmt.Errorf("while creating local checkpoint directory: %w", err) } + // An attempt that died after this rename, before the snapshot committed, + // left localDir behind. + if err := root.RemoveAll(localDir); err != nil { + return fmt.Errorf("while removing uncommitted local snapshot: %w", err) + } if err := root.Rename(stageDir, localDir); err != nil { return fmt.Errorf("failed to move %s to %s: %w", stageDir, localDir, err) } diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index 8cde90bf9c..7f2ec9ff81 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -787,7 +787,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke } } - ctx, cancel := context.WithCancel(ctx) + ctx, cancel := ateworker.CheckpointContext(ctx) defer cancel() // Not cancelable: a checkpoint is saving the actor's state. defer s.inFlight.Add(req.GetActorUid(), rpcCheckpointWorkload, nil)() diff --git a/cmd/ateworker-microvm/checkpoint.go b/cmd/ateworker-microvm/checkpoint.go index 4686651f98..ff1e178340 100644 --- a/cmd/ateworker-microvm/checkpoint.go +++ b/cmd/ateworker-microvm/checkpoint.go @@ -86,7 +86,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke } } - ctx, cancel := context.WithCancel(ctx) + ctx, cancel := ateworker.CheckpointContext(ctx) defer cancel() defer s.inFlight.Add(req.GetActorUid(), rpcCheckpointWorkload, nil)() diff --git a/internal/ateworker/checkpoint.go b/internal/ateworker/checkpoint.go index 98b4520e5a..66611a3364 100644 --- a/internal/ateworker/checkpoint.go +++ b/internal/ateworker/checkpoint.go @@ -15,6 +15,7 @@ package ateworker import ( + "context" "encoding/json" "os" "path/filepath" @@ -23,6 +24,18 @@ import ( "go.ate.dev/substrate/internal/proto/ateworkerpb" ) +// CheckpointContext returns the context a CheckpointWorkload runs in once it +// holds the actor's lock. It keeps ctx's values and deadline but not its +// cancellation, so atelet going away does not cancel a checkpoint that may +// already have stopped the workload. +func CheckpointContext(ctx context.Context) (context.Context, context.CancelFunc) { + detached := context.WithoutCancel(ctx) + if deadline, ok := ctx.Deadline(); ok { + return context.WithDeadline(detached, deadline) + } + return context.WithCancel(detached) +} + // checkpointDoneFile is written into checkpointDir after all snapshot files are // listed and the workload is torn down. It is wiped by os.RemoveAll(checkpointDir) // on the next checkpoint or atelet's resetActorDirs. diff --git a/internal/ateworker/checkpoint_test.go b/internal/ateworker/checkpoint_test.go index 8343de7a13..2ae265141e 100644 --- a/internal/ateworker/checkpoint_test.go +++ b/internal/ateworker/checkpoint_test.go @@ -15,10 +15,12 @@ package ateworker import ( + "context" "os" "path/filepath" "slices" "testing" + "time" "go.ate.dev/substrate/internal/proto/ateworkerpb" ) @@ -122,3 +124,38 @@ func TestReuseCheckpointDataSnapshotFilesByFidelity(t *testing.T) { t.Fatalf("ReuseCheckpoint(VOLUMES with non-data files) = %v, want nil", got) } } + +// TestCheckpointContext checks that a checkpoint outlives its canceled caller +// but keeps the caller's deadline. +func TestCheckpointContext(t *testing.T) { + deadline := time.Now().Add(time.Hour) + for _, tc := range []struct { + name string + caller func(context.Context) (context.Context, context.CancelFunc) + wantDeadline time.Time + }{ + { + name: "caller deadline", + caller: func(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithDeadline(ctx, deadline) + }, + wantDeadline: deadline, + }, + {name: "no deadline", caller: context.WithCancel}, + } { + t.Run(tc.name, func(t *testing.T) { + caller, cancelCaller := tc.caller(t.Context()) + ctx, cancel := CheckpointContext(caller) + defer cancel() + + cancelCaller() + if err := ctx.Err(); err != nil { + t.Errorf("Err() after the caller canceled = %v, want nil", err) + } + got, ok := ctx.Deadline() + if wantOK := !tc.wantDeadline.IsZero(); ok != wantOK || !got.Equal(tc.wantDeadline) { + t.Errorf("Deadline() = %v, %t, want %v, %t", got, ok, tc.wantDeadline, wantOK) + } + }) + } +} From 2e931ec5c07c6e06fa98d6c289457c7f07188a9d Mon Sep 17 00:00:00 2001 From: haiyanmeng Date: Tue, 27 Oct 2026 22:47:41 +0000 Subject: [PATCH 20/53] Rename --cordon-control-plane to --cordon-system-infra-components and its node pool label to ate-system-infra (#3210) `--cordon-control-plane` also pins atenet-router and atenet-egress, which are data plane components, so the name is confusing. Renamed: | Old | New | |---|---| | `--cordon-control-plane` | `--cordon-system-infra-components` | | `ATE_INSTALL_CORDON_CONTROL_PLANE` | `ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS` | | `cordonControlPlane` (config file key) | `cordonSystemInfraComponents` | | `ate.dev/workloadType=ate-control-plane` (shared pool label and taint) | `ate.dev/workloadType=ate-system-infra` | | `manifests/ate-install/components/cordon-control-plane` | `manifests/ate-install/components/cordon-system-infra-components` | The postgres pool label `ate.dev/workloadType=ate-postgres` is unchanged. The missing postgres pool error linked to `docs/upgrade.md#cordoned-control-plane-postgres-pool`, which doesn't exist. It now links to the System Infra Isolation section in `docs/architecture.md`. AI use: I used Claude Code to make the rename and update the docs and comments, and I reviewed the diff. ## Breaking change Installs that don't use `--cordon-control-plane` need no action. Installs that do: 1. Replace the old flag, env var or config file key with the new one in your scripts and config. The old flag and config key fail with an error. **The old env var is ignored silently**, and the install would stop cordoning. 2. Relabel and retaint the shared pool from `ate-control-plane` to `ate-system-infra`. Running pods keep running, because node selectors and `NoSchedule` taints only apply at scheduling. On GKE: ```bash gcloud container node-pools update --cluster $CLUSTER --location $LOCATION \ --node-labels ate.dev/workloadType=ate-system-infra gcloud container node-pools update --cluster $CLUSTER --location $LOCATION \ --node-taints ate.dev/workloadType=ate-system-infra:NoSchedule ``` Both flags replace the pool's whole set of labels or taints, so include any others the pool carries. gcloud asks for confirmation. Add `--quiet` to skip it. GKE updates the existing nodes in place. On other providers, update the pool's node template the same way. For nodes labeled by hand, overwrite the label and taint in place: ```bash kubectl label node ate.dev/workloadType=ate-system-infra --overwrite kubectl taint node ate.dev/workloadType=ate-system-infra:NoSchedule --overwrite ``` Don't remove the old taint before adding the new one. While the node has no taint, DaemonSets such as atelet schedule pods onto it. 3. Redeploy with `ate-setup deploy ate-system --cordon-system-infra-components`. The new pods schedule onto the relabeled pool. Do steps 2 and 3 close together. A system infra pod rescheduled between them stays Pending until step 3. - [x] Tests pass - [x] Docs updated --------- Signed-off-by: Haiyan Meng --- benchmarking/automation/controlplane.py | 5 +- benchmarking/automation/tests.yaml | 9 ++-- cmd/ate-setup/commands.md | 2 +- cmd/ate-setup/differences.md | 7 +-- cmd/ate-setup/internal/cmd/deploy.go | 2 +- cmd/ate-setup/internal/config/config.go | 20 ++++---- cmd/ate-setup/internal/config/config_test.go | 20 ++++---- cmd/ate-setup/internal/config/load.go | 2 +- cmd/ate-setup/internal/config/options_test.go | 4 +- cmd/ate-setup/internal/config/setting.go | 6 +-- cmd/ate-setup/internal/steps/overlay.go | 12 ++--- cmd/ate-setup/internal/steps/overlay_test.go | 51 ++++++++++--------- cmd/ate-setup/internal/steps/postgres.go | 33 ++++++------ cmd/ate-setup/internal/steps/postgres_test.go | 22 ++++---- docs/architecture.md | 18 ++++--- docs/operator-install.md | 2 +- .../kustomization.yaml | 19 ++++--- 17 files changed, 121 insertions(+), 113 deletions(-) rename manifests/ate-install/components/{cordon-control-plane => cordon-system-infra-components}/kustomization.yaml (83%) diff --git a/benchmarking/automation/controlplane.py b/benchmarking/automation/controlplane.py index 1b4003913d..75f0c964d0 100644 --- a/benchmarking/automation/controlplane.py +++ b/benchmarking/automation/controlplane.py @@ -17,8 +17,9 @@ (see API_SERVER_KNOBS) and a `postgres:` section (see POSTGRES_DEFAULTS). A test type validates the block with validate() and calls shape() from its -pre_test. Isolation from the load is the install's: --cordon-control-plane -in ateArgs keeps ate-api-server and Postgres on their own tainted pools. +pre_test. Isolation from the load is the install's: +--cordon-system-infra-components in ateArgs keeps ate-api-server and Postgres +on their own tainted pools. """ import json diff --git a/benchmarking/automation/tests.yaml b/benchmarking/automation/tests.yaml index 42e13cff07..a9d43827b7 100644 --- a/benchmarking/automation/tests.yaml +++ b/benchmarking/automation/tests.yaml @@ -128,8 +128,9 @@ # args Optional. Postgres settings as name=value (e.g. # commit_delay=100), applied in the restart stats makes. # The block leaves placement to the install. A capacity run sets -# --cordon-control-plane in ateArgs, which keeps ate-api-server and Postgres -# off the nodes that run the load, on the tainted pools that flag expects. +# --cordon-system-infra-components in ateArgs, which keeps ate-api-server and +# Postgres off the nodes that run the load, on the tainted pools that flag +# expects. # # ---- type: nighthawk-ingress -------------------------------------------------------- # @@ -722,7 +723,7 @@ tests: isolate: delay: 150ms ateArgs: - - "--cordon-control-plane" + - "--cordon-system-infra-components" controlPlane: apiServer: replicas: 1 @@ -750,7 +751,7 @@ tests: isolate: delay: 150ms ateArgs: - - "--cordon-control-plane" + - "--cordon-system-infra-components" controlPlane: apiServer: replicas: 1 diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 267b790648..60665e3ae9 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -29,7 +29,7 @@ a pre-scan pass, so they may appear anywhere on its command line. | `--podcert-workers-per-signer N` | `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer | | `--cluster-size size0\|size10` | `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent | | `--ateapi-postgres-storage-class NAME` | `ATE_API_POSTGRES_STORAGE_CLASS=NAME` | StorageClass for the bundled PostgreSQL volume instead of the cluster default, such as `dynamic-rwo` on GKE so C4 and N4 nodes can attach it. An existing install must delete the postgres StatefulSet and its claim first | -| `--cordon-control-plane` | `--cordon-control-plane` | Keep the control plane off the worker nodes. Assumes a small shared pool labeled and tainted `ate.dev/workloadType=ate-control-plane:NoSchedule`, across which each workload's replicas are spread, and a one-node pool labeled and tainted `ate.dev/workloadType=ate-postgres:NoSchedule` for postgres alone. `ATE_INSTALL_CORDON_CONTROL_PLANE=true` when the flag is absent | +| `--cordon-system-infra-components` | `--cordon-system-infra-components` | Keep the system infra components, such as ate-api-server, atenet-router, atenet-egress and postgres, off the worker nodes. Assumes a small shared pool labeled and tainted `ate.dev/workloadType=ate-system-infra:NoSchedule`, across which each workload's replicas are spread, and a one-node pool labeled and tainted `ate.dev/workloadType=ate-postgres:NoSchedule` for postgres alone. `ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS=true` when the flag is absent | | `--fake-data-plane` | `--fake-data-plane` | Install the fake data plane for control-plane benchmarks: fake-atelet takes atelet's place in its DaemonSet, taking its objectstore-plugin's storage settings, ate-controller is scaled to zero, and fake-workersync honors every WorkerPool with fake Workers on the fake-atelet nodes. Reaches `deploy ate-system`, `deploy atelet`, and `deploy ate-controller`; only `deploy ate-system` switches a cluster between the real and fake data planes, and the other two refuse a data plane other than the installed one. `deploy ate-system --fake-data-plane` refuses a cluster whose WorkerPools have worker pods. fake-atelet runs with its defaults; a benchmark tunes it through the optional `fake-atelet-tuning` ConfigMap, whose `FAKE_ATELET_*` variables set its flags, and which no install creates or resets. Actors do not exist; never use it on a cluster that serves real actors. `ATE_INSTALL_FAKE_DATA_PLANE=true` when the flag is absent | | `--ingress-allowed-spiffe-ids IDS` | `--ingress-allowed-spiffe-ids IDS`, or `ATE_INGRESS_ALLOWED_SPIFFE_IDS=IDS` | Required by `deploy ate-system` and `deploy atenet` for both dataplanes. Comma-separated SPIFFE IDs (exact match) of the clients the ingress router lets reach actors; any listed client may reach any actor. Both dataplanes require a Pod Identity CA client certificate on HTTPS and TLS CONNECT. The SPIFFE URI SAN allowlist may be checked during or after TLS, including after CONNECT acceptance, but unlisted clients must be denied before actor routing. The podidentity signer issues `spiffe://cluster.local/ns//sa/` | | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 725594261f..12209ac94b 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -185,11 +185,12 @@ model, so NotFound is ignored — and so is a kind that no longer resolves, sinc teardown after the CRDs are gone must not fail. Beyond that, deletes are strict. -**Cluster profiles are new.** `--cluster-size` and `--cordon-control-plane` +**Cluster profiles are new.** `--cluster-size` and `--cordon-system-infra-components` postdate the shell installer, which the shim forwards them to. `size10` renders the `podcert-size10` overlay and resizes the bundled PostgreSQL from -`postgres-size10/postgres-config-patch.yaml`; `--cordon-control-plane` -composes the `cordon-control-plane` component over every control plane apply +`postgres-size10/postgres-config-patch.yaml`; +`--cordon-system-infra-components` composes the +`cordon-system-infra-components` component over every system infra apply path. The size10 PostgreSQL changes are made to the decoded objects before the one server-side apply, not patched on afterwards: one rollout, the rollout wait sees the resized pod, and there is no second field manager for a later apply to diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 76d3f12ff0..0d9f38c8eb 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -60,7 +60,7 @@ ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING or the ATE_API_POSTGRES_CLOUDSQL_* select an external database. Shape the install with the global --atenet-dataplane, --cluster-size, ---cordon-control-plane, and --fake-data-plane flags.`, +--cordon-system-infra-components, and --fake-data-plane flags.`, // Args runs before the root command loads the configuration, so only the // flag is readable here. Checking it keeps an unusable value from costing // a credential fetch, which is the common case; a value arriving from the diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 68d8c6a6fa..4d4c32b21e 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -203,12 +203,14 @@ type Config struct { // or size10. ClusterSize string - // CordonControlPlane keeps the control plane off the worker nodes - // (ATE_INSTALL_CORDON_CONTROL_PLANE). It assumes a small shared pool - // labeled and tainted ate.dev/workloadType=ate-control-plane:NoSchedule, - // and a one-node pool labeled and tainted - // ate.dev/workloadType=ate-postgres:NoSchedule for postgres alone. - CordonControlPlane bool + // CordonSystemInfraComponents keeps the system infra components, such as + // ate-api-server, atenet-router, atenet-egress and postgres, off the + // worker nodes (ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS). It assumes a + // small shared pool labeled and tainted + // ate.dev/workloadType=ate-system-infra:NoSchedule, and a one-node pool + // labeled and tainted ate.dev/workloadType=ate-postgres:NoSchedule for + // postgres alone. + CordonSystemInfraComponents bool // FakeDataPlane installs fake-atelet in place of atelet and // fake-workersync in place of ate-controller @@ -507,9 +509,9 @@ func (c *Config) ScriptEnv() []string { if len(c.PreviewGates) > 0 { merged["ATE_PREVIEW"] = strings.Join(c.PreviewGates, ",") } - delete(merged, "ATE_INSTALL_CORDON_CONTROL_PLANE") - if c.CordonControlPlane { - merged["ATE_INSTALL_CORDON_CONTROL_PLANE"] = "true" + delete(merged, "ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS") + if c.CordonSystemInfraComponents { + merged["ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS"] = "true" } delete(merged, "ATE_INSTALL_FAKE_DATA_PLANE") if c.FakeDataPlane { diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index b25ca310c7..86abcf1f7f 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -65,7 +65,7 @@ func loadEnv(t *testing.T) { "ATE_IMAGE_TAG", "ATE_INGRESS_ALLOWED_SPIFFE_IDS", "ATE_INSTALL_CLUSTER_SIZE", - "ATE_INSTALL_CORDON_CONTROL_PLANE", + "ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS", "ATE_INSTALL_FAKE_DATA_PLANE", "ATE_INSTALL_KIND", "ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER", @@ -129,8 +129,8 @@ func TestLoadDefaults(t *testing.T) { if cfg.ClusterSize != ClusterSizeSize0 { t.Errorf("ClusterSize = %q, want %q", cfg.ClusterSize, ClusterSizeSize0) } - if cfg.CordonControlPlane { - t.Error("CordonControlPlane = true, want false") + if cfg.CordonSystemInfraComponents { + t.Error("CordonSystemInfraComponents = true, want false") } } @@ -191,30 +191,30 @@ func TestLoadClusterSize(t *testing.T) { } } -func TestLoadCordonControlPlane(t *testing.T) { +func TestLoadCordonSystemInfraComponents(t *testing.T) { for _, tc := range []struct { name string opts Options env string want bool }{ - {name: "flag", opts: Options{CordonControlPlane: true}, want: true}, + {name: "flag", opts: Options{CordonSystemInfraComponents: true}, want: true}, {name: "environment true", env: "true", want: true}, {name: "environment false", env: "false", want: false}, } { t.Run(tc.name, func(t *testing.T) { loadEnv(t) - t.Setenv("ATE_INSTALL_CORDON_CONTROL_PLANE", tc.env) + t.Setenv("ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS", tc.env) cfg, err := Load(tc.opts) if err != nil { t.Fatalf("Load() error = %v", err) } - if cfg.CordonControlPlane != tc.want { - t.Errorf("CordonControlPlane = %v, want %v", cfg.CordonControlPlane, tc.want) + if cfg.CordonSystemInfraComponents != tc.want { + t.Errorf("CordonSystemInfraComponents = %v, want %v", cfg.CordonSystemInfraComponents, tc.want) } - _, exported := scriptEnvMap(t, cfg)["ATE_INSTALL_CORDON_CONTROL_PLANE"] + _, exported := scriptEnvMap(t, cfg)["ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS"] if exported != tc.want { - t.Errorf("ScriptEnv() exports ATE_INSTALL_CORDON_CONTROL_PLANE = %v, want %v", exported, tc.want) + t.Errorf("ScriptEnv() exports ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS = %v, want %v", exported, tc.want) } }) } diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index a85f7d64d5..2a59416b25 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -273,7 +273,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro PodcertWorkersPerSigner: podcertWorkers, EnvoyConcurrency: envoyConcurrency, ClusterSize: r.String("clusterSize"), - CordonControlPlane: r.Bool("cordonControlPlane"), + CordonSystemInfraComponents: r.Bool("cordonSystemInfraComponents"), FakeDataPlane: r.Bool("fakeDataPlane.enabled"), IngressAllowedSPIFFEIDs: splitList(r.String("ingressAllowedSPIFFEIDs")), AdditionalEgressExtprocService: r.String("atenet.egress.additionalExtprocService"), diff --git a/cmd/ate-setup/internal/config/options_test.go b/cmd/ate-setup/internal/config/options_test.go index 3e8fba4ffc..d59e31a51f 100644 --- a/cmd/ate-setup/internal/config/options_test.go +++ b/cmd/ate-setup/internal/config/options_test.go @@ -38,7 +38,7 @@ type Options struct { PodcertWorkersPerSigner int EnvoyConcurrency int ClusterSize string - CordonControlPlane bool + CordonSystemInfraComponents bool FakeDataPlane bool AdditionalEgressExtprocService string CredentialProvider string @@ -103,7 +103,7 @@ func optionsFlagSet(opts Options) (*pflag.FlagSet, error) { val bool }{ {"kind", opts.Kind}, - {"cordon-control-plane", opts.CordonControlPlane}, + {"cordon-system-infra-components", opts.CordonSystemInfraComponents}, {"fake-data-plane", opts.FakeDataPlane}, } { if kv.val { diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 97b67e8f11..bac2ee3afa 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -286,10 +286,10 @@ var Registry = []Setting{ Usage: "Coarse-grained sizing of Substrate: size0 or size10", }, { - Key: "cordonControlPlane", Env: "ATE_INSTALL_CORDON_CONTROL_PLANE", Flag: "cordon-control-plane", + Key: "cordonSystemInfraComponents", Env: "ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS", Flag: "cordon-system-infra-components", Kind: KindBool, Default: "false", - Usage: "Keep the control plane off the worker nodes. Assumes a small shared pool labeled " + - "and tainted ate.dev/workloadType=ate-control-plane:NoSchedule, and a one-node pool " + + Usage: "Keep the system infra components off the worker nodes. Assumes a small shared pool labeled " + + "and tainted ate.dev/workloadType=ate-system-infra:NoSchedule, and a one-node pool " + "labeled and tainted ate.dev/workloadType=ate-postgres:NoSchedule for postgres alone", }, { diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index aa15059888..ef658c097c 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -36,10 +36,10 @@ const ( installDir = "manifests/ate-install" ) -// cordonControlPlaneComponent is the kustomize component that pins each -// control plane workload to its own node, layered over every control plane -// apply under --cordon-control-plane. -const cordonControlPlaneComponent = installDir + "/components/cordon-control-plane" +// cordonSystemInfraComponent is the kustomize component that pins each +// system infra workload to its own node pool, layered over every system +// infra apply under --cordon-system-infra-components. +const cordonSystemInfraComponent = installDir + "/components/cordon-system-infra-components" // fakeDataPlaneComponent is the kustomize component that puts fake-atelet in // atelet's DaemonSet and fake-workersync in place of ate-controller, layered @@ -60,8 +60,8 @@ func (e *Env) components() []string { if e.Cfg.Router == config.RouterAgentgateway { components = append(components, e.Cfg.Path(agentgatewayStaticMTLSComponent)) } - if e.Cfg.CordonControlPlane { - components = append(components, e.Cfg.Path(cordonControlPlaneComponent)) + if e.Cfg.CordonSystemInfraComponents { + components = append(components, e.Cfg.Path(cordonSystemInfraComponent)) } return components } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 2fe3ae34e3..2a6e8dffb8 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -346,10 +346,10 @@ func TestSystemOverlayDefaultIsBase(t *testing.T) { } // pinnedWorkloads returns the Deployment and StatefulSet names in a rendered -// manifest that carry the cordon-control-plane node pinning, mapped to the -// pool each one selects, and every workload name seen. Only the shared pool -// spreads its replicas, and no pool requires anti-affinity: that would need a -// node per pod. +// manifest that carry the cordon-system-infra-components node pinning, mapped +// to the pool each one selects, and every workload name seen. Only the shared +// pool spreads its replicas, and no pool requires anti-affinity: that would +// need a node per pod. func pinnedWorkloads(t *testing.T, manifest []byte) (pinned map[string]string, all []string) { t.Helper() pinned = map[string]string{} @@ -393,12 +393,12 @@ func pinnedWorkloads(t *testing.T, manifest []byte) (pinned map[string]string, a if podSpec.Affinity["podAntiAffinity"] != nil { t.Errorf("workload %s has pod anti-affinity", obj.Metadata.Name) } - if spread := len(podSpec.TopologySpreadConstraints) > 0; spread != (pool == "ate-control-plane") { + if spread := len(podSpec.TopologySpreadConstraints) > 0; spread != (pool == "ate-system-infra") { t.Errorf("workload %s in pool %s: topology spread = %v", obj.Metadata.Name, pool, spread) } // The spread narrows on the app label; without it the workload would // spread against the whole pool instead of its own replicas. - if pool == "ate-control-plane" && obj.Spec.Template.Metadata.Labels["app"] == "" { + if pool == "ate-system-infra" && obj.Spec.Template.Metadata.Labels["app"] == "" { t.Errorf("workload %s has no app label to spread its replicas by", obj.Metadata.Name) } pinned[obj.Metadata.Name] = pool @@ -406,12 +406,13 @@ func pinnedWorkloads(t *testing.T, manifest []byte) (pinned map[string]string, a return pinned, all } -// Under --cordon-control-plane every control plane apply path has to carry the -// pinning, since each workload reaches the cluster through a different one: -// the system bundle, the lone redeploy files, the podcert overlay, the -// postgres file, the egress variants, and the bundled credential provider. postgres gets a pool of its own; -// every other control plane workload shares one. -func TestRenderCordonControlPlane(t *testing.T) { +// Under --cordon-system-infra-components every system infra apply path has +// to carry the pinning, since each workload reaches the cluster through a +// different one: the system bundle, the lone redeploy files, the podcert +// overlay, the postgres file, the egress variants, and the bundled credential +// provider. postgres gets a pool of its own; every other system infra +// workload shares one. +func TestRenderCordonSystemInfraComponents(t *testing.T) { root := repoRoot(t) for _, tc := range []struct { name string @@ -489,7 +490,7 @@ func TestRenderCordonControlPlane(t *testing.T) { t.Run(tc.name, func(t *testing.T) { cfg := tc.cfg cfg.Root = root - cfg.CordonControlPlane = true + cfg.CordonSystemInfraComponents = true e := &Env{Cfg: &cfg} rendered, err := e.render(tc.path(e)) @@ -501,7 +502,7 @@ func TestRenderCordonControlPlane(t *testing.T) { if !slices.Contains(all, name) { t.Errorf("rendered manifest has no workload %s (found %v)", name, all) } - want := "ate-control-plane" + want := "ate-system-infra" if name == "postgres" { want = "ate-postgres" } @@ -510,10 +511,10 @@ func TestRenderCordonControlPlane(t *testing.T) { } } // The DaemonSet-shaped and demo workloads stay off the pools; only - // the named control plane workloads are pinned. + // the named system infra workloads are pinned. for name := range pinned { if !slices.Contains(tc.want, name) { - t.Errorf("workload %s is pinned but is not a control plane workload", name) + t.Errorf("workload %s is pinned but is not a system infra workload", name) } } }) @@ -522,10 +523,10 @@ func TestRenderCordonControlPlane(t *testing.T) { // The extproc-patched egress manifest arrives as bytes, and the pinning has to // reach it too. -func TestRenderBytesCordonControlPlane(t *testing.T) { +func TestRenderBytesCordonSystemInfraComponents(t *testing.T) { e := &Env{Cfg: &config.Config{ Root: repoRoot(t), - CordonControlPlane: true, + CordonSystemInfraComponents: true, AdditionalEgressExtprocService: "ate-system/foo:50051", }} patched, err := e.patchAtenetEgressManifest() @@ -537,7 +538,7 @@ func TestRenderBytesCordonControlPlane(t *testing.T) { t.Fatalf("renderBytes: %v", err) } pinned, _ := pinnedWorkloads(t, rendered) - if pinned["atenet-egress"] != "ate-control-plane" { + if pinned["atenet-egress"] != "ate-system-infra" { t.Errorf("atenet-egress is not pinned in the composed extproc manifest (pinned: %v)", pinned) } if !strings.Contains(string(rendered), additionalEgressExtprocCluster) { @@ -553,14 +554,14 @@ func TestRenderWithoutCordonLeavesManifestsAlone(t *testing.T) { t.Fatalf("render: %v", err) } if pinned, _ := pinnedWorkloads(t, rendered); len(pinned) != 0 { - t.Errorf("render without --cordon-control-plane pinned %v", pinned) + t.Errorf("render without --cordon-system-infra-components pinned %v", pinned) } raw, err := os.ReadFile(e.Cfg.Manifest("ate-api-server.yaml")) if err != nil { t.Fatal(err) } if string(rendered) != string(raw) { - t.Error("render of a plain file without --cordon-control-plane is not the file's bytes") + t.Error("render of a plain file without --cordon-system-infra-components is not the file's bytes") } } @@ -652,7 +653,7 @@ func TestRenderFakeDataPlane(t *testing.T) { }, { name: "controller file under cordon", - cfg: config.Config{CordonControlPlane: true}, + cfg: config.Config{CordonSystemInfraComponents: true}, path: func(e *Env) string { return e.Cfg.Manifest("ate-controller.yaml") }, wantController: true, }, @@ -756,11 +757,11 @@ func TestRenderFakeDataPlane(t *testing.T) { // Both components compose: the cordon pinning survives the fake // one, and reaches fake-workersync, which stands in for // ate-controller. - if cfg.CordonControlPlane { + if cfg.CordonSystemInfraComponents { pinned, _ := pinnedWorkloads(t, rendered) for _, name := range []string{"ate-controller", fakeWorkersyncDeployment} { - if pinned[name] != "ate-control-plane" { - t.Errorf("under --cordon-control-plane %s is pinned to %q, want ate-control-plane", name, pinned[name]) + if pinned[name] != "ate-system-infra" { + t.Errorf("under --cordon-system-infra-components %s is pinned to %q, want ate-system-infra", name, pinned[name]) } } } diff --git a/cmd/ate-setup/internal/steps/postgres.go b/cmd/ate-setup/internal/steps/postgres.go index 3e8be19b4a..d3544adf7f 100644 --- a/cmd/ate-setup/internal/steps/postgres.go +++ b/cmd/ate-setup/internal/steps/postgres.go @@ -90,21 +90,21 @@ func (e *Env) waitAndSetupBundledPostgres(ctx context.Context) error { } // The size10 PostgreSQL container. Deliberately no CPU limit: under -// --cordon-control-plane the dedicated ate-postgres pool keeps the pod alone -// on its node, so a limit would only add CFS throttling on checkpoint and -// autovacuum bursts. Without that flag the pod shares whatever node fits the -// request, and the missing limit lets it contend with its neighbors. Memory -// request == limit keeps eviction ordering equivalent to a Guaranteed pod, -// which matters because postgres cannot release shared_buffers under pressure. -// postgres-size10/postgres-config-patch.yaml is tuned to these numbers, so they -// move together. +// --cordon-system-infra-components the dedicated ate-postgres pool keeps the +// pod alone on its node, so a limit would only add CFS throttling on +// checkpoint and autovacuum bursts. Without that flag the pod shares whatever +// node fits the request, and the missing limit lets it contend with its +// neighbors. Memory request == limit keeps eviction ordering equivalent to a +// Guaranteed pod, which matters because postgres cannot release +// shared_buffers under pressure. postgres-size10/postgres-config-patch.yaml +// is tuned to these numbers, so they move together. const ( size10PostgresCPURequest = "80" size10PostgresMemory = "140Gi" ) -// postgresPoolSelector selects the nodes of the pool the cordon-control-plane -// component pins the bundled PostgreSQL to. +// postgresPoolSelector selects the nodes of the pool the +// cordon-system-infra-components component pins the bundled PostgreSQL to. const postgresPoolSelector = "ate.dev/workloadType=ate-postgres" // postgresPlan says where ateapi's store comes from: the bundled StatefulSet, @@ -242,11 +242,12 @@ func setPostgresStorageClass(objs []*unstructured.Unstructured, class string) er } // requirePostgresPool refuses to apply the bundled PostgreSQL under -// --cordon-control-plane when no node carries the ate-postgres label. The -// StatefulSet deletes its running pod before creating the replacement, so -// without the pool the database goes down and the new pod stays Pending. +// --cordon-system-infra-components when no node carries the ate-postgres +// label. The StatefulSet deletes its running pod before creating the +// replacement, so without the pool the database goes down and the new pod +// stays Pending. func (e *Env) requirePostgresPool(ctx context.Context) error { - if !e.Cfg.CordonControlPlane { + if !e.Cfg.CordonSystemInfraComponents { return nil } nodes, err := e.Kube.Typed.CoreV1().Nodes().List(ctx, metav1.ListOptions{ @@ -257,9 +258,9 @@ func (e *Env) requirePostgresPool(ctx context.Context) error { return fmt.Errorf("while listing the %s nodes: %w", postgresPoolSelector, err) } if len(nodes.Items) == 0 { - return fmt.Errorf("--cordon-control-plane pins the bundled PostgreSQL to nodes labeled %s, and the cluster has none: "+ + return fmt.Errorf("--cordon-system-infra-components pins the bundled PostgreSQL to nodes labeled %s, and the cluster has none: "+ "create a one-node pool labeled and tainted %s:NoSchedule in the zone of the postgres volume first "+ - "(see docs/upgrade.md#cordoned-control-plane-postgres-pool)", postgresPoolSelector, postgresPoolSelector) + "(see docs/architecture.md#system-infra-isolation)", postgresPoolSelector, postgresPoolSelector) } return nil } diff --git a/cmd/ate-setup/internal/steps/postgres_test.go b/cmd/ate-setup/internal/steps/postgres_test.go index a9bfdc1cd3..715c6e68c7 100644 --- a/cmd/ate-setup/internal/steps/postgres_test.go +++ b/cmd/ate-setup/internal/steps/postgres_test.go @@ -315,11 +315,11 @@ func labeledNode(name string, labels map[string]string) runtime.Object { return &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels}} } -// Under --cordon-control-plane the StatefulSet's rolling update deletes the -// running postgres before its replacement schedules, so a cluster with no -// ate-postgres node must be refused before anything is applied. +// Under --cordon-system-infra-components the StatefulSet's rolling update +// deletes the running postgres before its replacement schedules, so a cluster +// with no ate-postgres node must be refused before anything is applied. func TestRequirePostgresPool(t *testing.T) { - controlPlane := labeledNode("cp-1", map[string]string{"ate.dev/workloadType": "ate-control-plane"}) + systemInfra := labeledNode("infra-1", map[string]string{"ate.dev/workloadType": "ate-system-infra"}) postgres := labeledNode("pg-1", map[string]string{"ate.dev/workloadType": "ate-postgres"}) for _, tc := range []struct { name string @@ -327,22 +327,22 @@ func TestRequirePostgresPool(t *testing.T) { nodes []runtime.Object wantErr bool }{ - {name: "not cordoned", nodes: []runtime.Object{controlPlane}}, - {name: "cordoned with a postgres node", cordon: true, nodes: []runtime.Object{controlPlane, postgres}}, - {name: "cordoned without a postgres node", cordon: true, nodes: []runtime.Object{controlPlane}, wantErr: true}, + {name: "not cordoned", nodes: []runtime.Object{systemInfra}}, + {name: "cordoned with a postgres node", cordon: true, nodes: []runtime.Object{systemInfra, postgres}}, + {name: "cordoned without a postgres node", cordon: true, nodes: []runtime.Object{systemInfra}, wantErr: true}, {name: "cordoned with no nodes", cordon: true, wantErr: true}, } { t.Run(tc.name, func(t *testing.T) { e := &Env{ - Cfg: &config.Config{CordonControlPlane: tc.cordon}, + Cfg: &config.Config{CordonSystemInfraComponents: tc.cordon}, Kube: fakeKube(t, tc.nodes...), } err := e.requirePostgresPool(t.Context()) if gotErr := err != nil; gotErr != tc.wantErr { t.Fatalf("requirePostgresPool() error = %v, wantErr %v", err, tc.wantErr) } - if err != nil && !strings.Contains(err.Error(), "docs/upgrade.md") { - t.Errorf("requirePostgresPool() error = %q, want it to point at docs/upgrade.md", err) + if err != nil && !strings.Contains(err.Error(), "docs/architecture.md") { + t.Errorf("requirePostgresPool() error = %q, want it to point at docs/architecture.md", err) } }) } @@ -352,7 +352,7 @@ func TestRequirePostgresPool(t *testing.T) { // take to the StatefulSet, so the pool check has to sit in front of it. func TestApplyPostgresRequiresPostgresPool(t *testing.T) { e := &Env{ - Cfg: &config.Config{Root: repoRoot(t), CordonControlPlane: true}, + Cfg: &config.Config{Root: repoRoot(t), CordonSystemInfraComponents: true}, Kube: fakeKube(t), } err := e.applyPostgres(t.Context()) diff --git a/docs/architecture.md b/docs/architecture.md index 98d6644132..0dc38cc4e0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -371,17 +371,19 @@ Handles actor-aware routing and automatic re-animation. * **Latency**: The data plane is optimized for sub-100ms activation by bypassing Kubernetes' eventual consistency and performing atomic physical assignments. -### Control Plane Isolation +### System Infra Isolation -Control plane components can be kept on nodes of their own, -away from the workers, so a burst of worker or actor load cannot starve them. -When configured via the installer, the control plane workloads select a -node label and tolerate the matching `NoSchedule` taint: +System infra components, such as ate-api-server, atenet-router, atenet-egress +and postgres, can be kept on nodes of their own, away from the workers, so a +burst of worker or actor load cannot starve them. Under +`--cordon-system-infra-components`, these workloads select a node label and +tolerate the matching `NoSchedule` taint: -* `ate.dev/workloadType=ate-control-plane`: a small pool shared by every - control plane workload except postgres. Each workload's replicas are spread +* `ate.dev/workloadType=ate-system-infra`: a small pool shared by every + system infra workload except postgres. Each workload's replicas are spread across its nodes on a best-effort basis. -* `ate.dev/workloadType=ate-postgres`: a one-node pool for postgres alone. +* `ate.dev/workloadType=ate-postgres`: a one-node pool for postgres alone, + in the zone of the postgres volume. ## Actor Lifecycle diff --git a/docs/operator-install.md b/docs/operator-install.md index 86b46f6d5b..b32ed3d0a9 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -296,7 +296,7 @@ demos, the [benchmarks](../benchmarking/README.md#settings), and the | `atenet.egress.credentialProvider` | `ATE_CREDENTIAL_PROVIDER` | `--credential-provider` | — | (global) | | `clusterSize` | `ATE_INSTALL_CLUSTER_SIZE` | `--cluster-size` | `size0` | (global) | | `context` | `KUBECTL_CONTEXT` | `--context` | — | (global) | -| `cordonControlPlane` | `ATE_INSTALL_CORDON_CONTROL_PLANE` | `--cordon-control-plane` | `false` | (global) | +| `cordonSystemInfraComponents` | `ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS` | `--cordon-system-infra-components` | `false` | (global) | | `csi.setup` | `SETUP_CSI` | `--setup-csi` | `none` | `deploy ate-system` | | `docker.buildFlags` | `DOCKER_BUILD_FLAGS` | `--docker-build-flags` | — | (global) | | `fakeDataPlane.enabled` | `ATE_INSTALL_FAKE_DATA_PLANE` | `--fake-data-plane` | `false` | (global) | diff --git a/manifests/ate-install/components/cordon-control-plane/kustomization.yaml b/manifests/ate-install/components/cordon-system-infra-components/kustomization.yaml similarity index 83% rename from manifests/ate-install/components/cordon-control-plane/kustomization.yaml rename to manifests/ate-install/components/cordon-system-infra-components/kustomization.yaml index edc70647dd..e1a84e24de 100644 --- a/manifests/ate-install/components/cordon-control-plane/kustomization.yaml +++ b/manifests/ate-install/components/cordon-system-infra-components/kustomization.yaml @@ -12,16 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Keeps the control plane off the worker nodes. Selected by -# hack/install-ate.sh and cmd/ate-setup under --cordon-control-plane; both -# wrap every control plane apply in a kustomization that includes this -# component. +# Keeps the system infra components off the worker nodes. Selected by +# cmd/ate-setup under --cordon-system-infra-components, which wraps every +# system infra apply in a kustomization that includes this component. # # Assumes two node pools, each carrying its label as a matching NoSchedule # taint: # -# ate.dev/workloadType=ate-control-plane A small pool shared by every -# control plane workload except postgres. Each workload's replicas are +# ate.dev/workloadType=ate-system-infra A small pool shared by every +# system infra workload except postgres. Each workload's replicas are # spread across its nodes on a best-effort basis, so the pool may be # smaller than the pod count and rollout surges never wait on a free # node. @@ -52,14 +51,14 @@ patches: template: metadata: labels: - ate.dev/workloadType: ate-control-plane + ate.dev/workloadType: ate-system-infra spec: nodeSelector: - ate.dev/workloadType: ate-control-plane + ate.dev/workloadType: ate-system-infra tolerations: - key: ate.dev/workloadType operator: Equal - value: ate-control-plane + value: ate-system-infra effect: NoSchedule topologySpreadConstraints: # matchLabelKeys narrows the selector to pods sharing this pod's @@ -69,7 +68,7 @@ patches: whenUnsatisfiable: ScheduleAnyway labelSelector: matchLabels: - ate.dev/workloadType: ate-control-plane + ate.dev/workloadType: ate-system-infra matchLabelKeys: - app - target: From b5d7201443c21860d84878098a668d767bd1dae3 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 27 Oct 2026 23:43:15 +0000 Subject: [PATCH 21/53] ate-setup: separate e2e kustomize component from kind overlay (#3318) Move atelet's e2e-specific snapshot upload delay out of the kind overlay into a dedicated kustomize component at manifests/ate-install/components/e2e, selected with --e2e (or ATE_INSTALL_E2E=true / config.e2e.enabled). This keeps regular kind development and production installs on the default atelet snapshot timings while allowing both kind and other e2e runs to opt into the shorter e2e delays. AI use: AI-assisted, I reviewed and tested it. - [x] Tests pass - [x] Docs updated Signed-off-by: msau42 --- .github/workflows/pr-workflow.yaml | 3 + cmd/ate-setup/internal/config/config.go | 8 ++ cmd/ate-setup/internal/config/config_test.go | 29 +++++ cmd/ate-setup/internal/config/load.go | 1 + cmd/ate-setup/internal/config/options_test.go | 2 + cmd/ate-setup/internal/config/setting.go | 5 + cmd/ate-setup/internal/steps/overlay.go | 10 ++ cmd/ate-setup/internal/steps/overlay_test.go | 117 ++++++++++++++++++ hack/run-e2e.sh | 4 + internal/e2e/README.md | 24 ++-- .../components/e2e/kustomization.yaml | 39 ++++++ .../kind/atelet/kustomization.yaml | 3 - 12 files changed, 233 insertions(+), 12 deletions(-) create mode 100644 manifests/ate-install/components/e2e/kustomization.yaml diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 4584f1057b..00940a6e29 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -191,6 +191,9 @@ jobs: preview: ['off', 'on'] env: E2E_ATENET_DATAPLANE: ${{ matrix.dataplane }} + # Apply the e2e test component (short atelet snapshot delays) across the + # initial install and the control plane redeploy in run-microvm-demo-kind.sh. + ATE_INSTALL_E2E_TUNING: "true" # The installer requires a credential provider selection, and deploys the # bundled Kubernetes Secrets provider with the control plane. Job-wide # because every ate-setup invocation reads it, including the control diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 4d4c32b21e..e5de83a236 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -217,6 +217,10 @@ type Config struct { // (ATE_INSTALL_FAKE_DATA_PLANE), for control-plane benchmarks. FakeDataPlane bool + // E2ETuning layers the e2e test kustomize component over control plane and + // data plane applies (ATE_INSTALL_E2E_TUNING). + E2ETuning bool + // IngressAllowedSPIFFEIDs are the SPIFFE IDs the ingress router lets // reach actors over its TLS listeners (ATE_INGRESS_ALLOWED_SPIFFE_IDS). // See RequireIngressAllowedSPIFFEIDs. @@ -517,6 +521,10 @@ func (c *Config) ScriptEnv() []string { if c.FakeDataPlane { merged["ATE_INSTALL_FAKE_DATA_PLANE"] = "true" } + delete(merged, "ATE_INSTALL_E2E_TUNING") + if c.E2ETuning { + merged["ATE_INSTALL_E2E_TUNING"] = "true" + } delete(merged, "ATE_INGRESS_ALLOWED_SPIFFE_IDS") if len(c.IngressAllowedSPIFFEIDs) > 0 { merged["ATE_INGRESS_ALLOWED_SPIFFE_IDS"] = strings.Join(c.IngressAllowedSPIFFEIDs, ",") diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 86abcf1f7f..58f87c9338 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -66,6 +66,7 @@ func loadEnv(t *testing.T) { "ATE_INGRESS_ALLOWED_SPIFFE_IDS", "ATE_INSTALL_CLUSTER_SIZE", "ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS", + "ATE_INSTALL_E2E_TUNING", "ATE_INSTALL_FAKE_DATA_PLANE", "ATE_INSTALL_KIND", "ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER", @@ -248,6 +249,34 @@ func TestLoadFakeDataPlane(t *testing.T) { } } +func TestLoadE2ETuning(t *testing.T) { + for _, tc := range []struct { + name string + opts Options + env string + want bool + }{ + {name: "off by default"}, + {name: "flag", opts: Options{E2ETuning: true}, want: true}, + {name: "environment", env: "true", want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_INSTALL_E2E_TUNING", tc.env) + cfg, err := Load(tc.opts) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.E2ETuning != tc.want { + t.Errorf("E2ETuning = %v, want %v", cfg.E2ETuning, tc.want) + } + if _, exported := scriptEnvMap(t, cfg)["ATE_INSTALL_E2E_TUNING"]; exported != tc.want { + t.Errorf("ScriptEnv() exports ATE_INSTALL_E2E_TUNING = %v, want %v", exported, tc.want) + } + }) + } +} + func TestLoadIngressAllowedSPIFFEIDs(t *testing.T) { const a, b = "spiffe://cluster.local/ns/a/sa/a", "spiffe://cluster.local/ns/b/sa/b" for _, tc := range []struct { diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 2a59416b25..fa16bd5176 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -275,6 +275,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro ClusterSize: r.String("clusterSize"), CordonSystemInfraComponents: r.Bool("cordonSystemInfraComponents"), FakeDataPlane: r.Bool("fakeDataPlane.enabled"), + E2ETuning: r.Bool("e2e.tuning"), IngressAllowedSPIFFEIDs: splitList(r.String("ingressAllowedSPIFFEIDs")), AdditionalEgressExtprocService: r.String("atenet.egress.additionalExtprocService"), CredentialProviderJSON: r.String("atenet.egress.credentialProvider"), diff --git a/cmd/ate-setup/internal/config/options_test.go b/cmd/ate-setup/internal/config/options_test.go index d59e31a51f..9f99a2545f 100644 --- a/cmd/ate-setup/internal/config/options_test.go +++ b/cmd/ate-setup/internal/config/options_test.go @@ -40,6 +40,7 @@ type Options struct { ClusterSize string CordonSystemInfraComponents bool FakeDataPlane bool + E2ETuning bool AdditionalEgressExtprocService string CredentialProvider string OtlpEndpoint string @@ -105,6 +106,7 @@ func optionsFlagSet(opts Options) (*pflag.FlagSet, error) { {"kind", opts.Kind}, {"cordon-system-infra-components", opts.CordonSystemInfraComponents}, {"fake-data-plane", opts.FakeDataPlane}, + {"e2e-tuning", opts.E2ETuning}, } { if kv.val { if err := fs.Set(kv.flag, "true"); err != nil { diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index bac2ee3afa..aa2b12181c 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -269,6 +269,11 @@ var Registry = []Setting{ Flag: "podcert-workers-per-signer", Kind: KindInt, Default: "0", Usage: "Worker goroutines per signer in podcertificate-controller", }, + { + Key: "e2e.tuning", Env: "ATE_INSTALL_E2E_TUNING", Flag: "e2e-tuning", + Kind: KindBool, Default: "false", + Usage: "Apply the e2e test component (short atelet snapshot delays for end-to-end test runs)", + }, { Key: "e2e.envoyConcurrency", Env: "E2E_ATE_ENVOY_CONCURRENCY", Flag: "e2e-envoy-concurrency", Kind: KindInt, Default: "0", diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index ef658c097c..78ef2f4596 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -41,6 +41,11 @@ const ( // infra apply under --cordon-system-infra-components. const cordonSystemInfraComponent = installDir + "/components/cordon-system-infra-components" +// e2eComponent is the kustomize component that tunes the control plane and +// data plane for end-to-end test runs, layered over every control plane apply +// under --e2e-tuning. +const e2eComponent = installDir + "/components/e2e" + // fakeDataPlaneComponent is the kustomize component that puts fake-atelet in // atelet's DaemonSet and fake-workersync in place of ate-controller, layered // over the atelet and ate-controller applies under --fake-data-plane. @@ -54,6 +59,11 @@ const agentgatewayStaticMTLSComponent = installDir + "/components/agentgateway-s // carries none of them is harmless. func (e *Env) components() []string { var components []string + // e2e must come before fake-data-plane: e2e appends to atelet's args, + // and fake-data-plane removes args entirely. + if e.Cfg.E2ETuning { + components = append(components, e.Cfg.Path(e2eComponent)) + } if e.Cfg.FakeDataPlane { components = append(components, e.Cfg.Path(fakeDataPlaneComponent)) } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 2a6e8dffb8..066cbafb23 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -769,6 +769,123 @@ func TestRenderFakeDataPlane(t *testing.T) { } } +// Under --e2e-tuning every path that applies atelet appends the e2e tuning +// flags without dropping the overlay's own args, and a non-e2e kind render +// keeps atelet on its default snapshot upload delay. +func TestRenderE2ETuning(t *testing.T) { + root := repoRoot(t) + const ateletDS = "atelet" + for _, tc := range []struct { + name string + cfg config.Config + path func(e *Env) string + wantAtelet bool + wantArgs []string + }{ + { + name: "base bundle", + cfg: config.Config{Router: config.RouterEnvoy}, + path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, + wantAtelet: true, + wantArgs: []string{"--grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem", "--snapshot-upload-delay=2s"}, + }, + { + name: "kind bundle", + cfg: config.Config{Router: config.RouterEnvoy, Kind: true}, + path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, + wantAtelet: true, + wantArgs: []string{"--localhost-registry-replacement=kind-registry:5000", "--log-level=debug", "--snapshot-upload-delay=2s"}, + }, + { + name: "agentgateway bundle", + cfg: config.Config{Router: config.RouterAgentgateway}, + path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, + wantAtelet: true, + wantArgs: []string{"--grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem", "--snapshot-upload-delay=2s"}, + }, + { + name: "kind agentgateway bundle", + cfg: config.Config{Router: config.RouterAgentgateway, Kind: true}, + path: func(e *Env) string { return e.Cfg.Path(SystemOverlay(e.Cfg)) }, + wantAtelet: true, + wantArgs: []string{"--localhost-registry-replacement=kind-registry:5000", "--log-level=debug", "--snapshot-upload-delay=2s"}, + }, + { + name: "atelet file", + path: func(e *Env) string { return e.Cfg.Manifest("atelet.yaml") }, + wantAtelet: true, + wantArgs: []string{"--grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem", "--snapshot-upload-delay=2s"}, + }, + { + name: "kind atelet overlay", + cfg: config.Config{Kind: true}, + path: func(e *Env) string { return e.ateletManifestPath() }, + wantAtelet: true, + wantArgs: []string{"--localhost-registry-replacement=kind-registry:5000", "--log-level=debug", "--snapshot-upload-delay=2s"}, + }, + { + name: "api server file without atelet", + path: func(e *Env) string { return e.Cfg.Manifest("ate-api-server.yaml") }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := tc.cfg + cfg.Root = root + cfg.E2ETuning = true + e := &Env{Cfg: &cfg} + + rendered, err := e.render(tc.path(e)) + if err != nil { + t.Fatalf("render: %v", err) + } + daemonSets, _ := dataPlaneWorkloads(t, rendered) + ds, ok := daemonSets[ateletDS] + if ok != tc.wantAtelet { + t.Fatalf("DaemonSet %s present = %v, want %v", ateletDS, ok, tc.wantAtelet) + } + if !ok { + return + } + args := ds.Spec.Template.Spec.Containers[0].Args + for _, want := range tc.wantArgs { + if !slices.Contains(args, want) { + t.Errorf("atelet args = %v, missing %q", args, want) + } + } + }) + } + + t.Run("kind without e2e-tuning omits snapshot-upload-delay", func(t *testing.T) { + e := &Env{Cfg: &config.Config{Root: root, Kind: true}} + rendered, err := e.render(e.ateletManifestPath()) + if err != nil { + t.Fatalf("render: %v", err) + } + daemonSets, _ := dataPlaneWorkloads(t, rendered) + args := daemonSets[ateletDS].Spec.Template.Spec.Containers[0].Args + if !slices.Contains(args, "--log-level=debug") { + t.Errorf("kind atelet args = %v, missing --log-level=debug", args) + } + for _, arg := range args { + if strings.HasPrefix(arg, "--snapshot-upload-delay") { + t.Errorf("kind atelet without --e2e-tuning carries %q in args %v", arg, args) + } + } + }) + + t.Run("composes with fake-data-plane", func(t *testing.T) { + e := &Env{Cfg: &config.Config{Root: root, E2ETuning: true, FakeDataPlane: true}} + rendered, err := e.render(e.ateletManifestPath()) + if err != nil { + t.Fatalf("render: %v", err) + } + daemonSets, _ := dataPlaneWorkloads(t, rendered) + if args := daemonSets[ateletDS].Spec.Template.Spec.Containers[0].Args; len(args) != 0 { + t.Errorf("fake-atelet args = %v, want none", args) + } + }) +} + // Both paths that deliver the Envoy router pass it the install's allowlist, // whatever the overlay, and refuse to render without one. func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh index 0667ab8e18..c2cdbde165 100755 --- a/hack/run-e2e.sh +++ b/hack/run-e2e.sh @@ -29,6 +29,10 @@ Usage: $0 [target-path] [go-test-flags] [-args [e2e-flags]] Runs End-to-End tests. +The target cluster must be installed with --e2e-tuning (or +ATE_INSTALL_E2E_TUNING=true) so atelet uses the shorter E2E snapshot delays +instead of its 5m production default. + The optional "target-path" must be the first argument and must start with "./internal/e2e" or "internal/e2e". It defaults to "./internal/e2e/suites/...". diff --git a/internal/e2e/README.md b/internal/e2e/README.md index 2e075470b5..01bdaf3e36 100644 --- a/internal/e2e/README.md +++ b/internal/e2e/README.md @@ -18,9 +18,10 @@ $ go test -v ./internal/e2e/suites/... -args --e2e ## Preconditions -The e2e tests assume you have a cluster set up with Agent Substrate installed, -for example via `hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}'` -or `hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}'`. +The e2e tests assume you have a cluster set up with Agent Substrate installed +with the `--e2e-tuning` component, for example via +`hack/install-ate.sh deploy ate-system --e2e-tuning --credential-provider='{"name":"k8s.io"}'` +or `hack/install-ate-kind.sh deploy ate-system --e2e-tuning --credential-provider='{"name":"k8s.io"}'`. The egress credential injection suite needs that bundled provider, which is what CI installs on the envoy lane; see `internal/e2e/suites/egresscredinject`. @@ -58,17 +59,22 @@ a checkpoint). ## Settings -The installer reads this setting in addition to those in +The installer reads these settings in addition to those in [docs/operator-install.md](../../docs/operator-install.md#settings), through the same -flag, environment variable, and configuration document channels. It sets the -worker thread count (`--concurrency`) of the Envoy container in `atenet-egress`; -`0` leaves the manifest unchanged. The -[nighthawk egress benchmark](../../benchmarking/nighthawk-egress/README.md) sets -it to `1` to measure single-CPU Envoy performance. +flag, environment variable, and configuration document channels: + +* `e2e.tuning` layers `manifests/ate-install/components/e2e` over control + plane and data plane applies, configuring short atelet snapshot delays for + end-to-end test runs. +* `e2e.envoyConcurrency` sets the worker thread count (`--concurrency`) of the + Envoy container in `atenet-egress`; `0` leaves the manifest unchanged. The + [nighthawk egress benchmark](../../benchmarking/nighthawk-egress/README.md) + sets it to `1` to measure single-CPU Envoy performance. | Key | Environment variable | Flag | Default | Applies to | |------------------------|-----------------------------|---------------------------|---------|------------| | `e2e.envoyConcurrency` | `E2E_ATE_ENVOY_CONCURRENCY` | `--e2e-envoy-concurrency` | `0` | (global) | +| `e2e.tuning` | `ATE_INSTALL_E2E_TUNING` | `--e2e-tuning` | `false` | (global) | ## Preview features diff --git a/manifests/ate-install/components/e2e/kustomization.yaml b/manifests/ate-install/components/e2e/kustomization.yaml new file mode 100644 index 0000000000..998ae28d2c --- /dev/null +++ b/manifests/ate-install/components/e2e/kustomization.yaml @@ -0,0 +1,39 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Tunes the control plane and data plane for end-to-end test runs. Selected by +# cmd/ate-setup under --e2e-tuning (or ATE_INSTALL_E2E_TUNING=true), which +# composes every control plane apply with this component. + +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +patches: + # Upload asynchronous snapshots in the background 2s after a suspend (5m by + # default), so e2e suites run alongside uploads. Appended via JSON6902 rather + # than a strategic merge patch so the base or kind overlay's existing args + # are preserved. + - target: + group: apps + version: v1 + kind: DaemonSet + name: atelet + namespace: ate-system + patch: |- + - op: test + path: /spec/template/spec/containers/0/name + value: atelet + - op: add + path: /spec/template/spec/containers/0/args/- + value: --snapshot-upload-delay=2s diff --git a/manifests/ate-install/kind/atelet/kustomization.yaml b/manifests/ate-install/kind/atelet/kustomization.yaml index b18e13a69e..0616973c6b 100644 --- a/manifests/ate-install/kind/atelet/kustomization.yaml +++ b/manifests/ate-install/kind/atelet/kustomization.yaml @@ -37,9 +37,6 @@ patches: # Kind clusters are dev/CI: run atelet at debug so e2e suites can # assert on per-item log lines. Production installs default to info. - --log-level=debug - # Upload asynchronous snapshots in the background 2s after a - # suspend (5m by default), so e2e suites run alongside uploads. - - --snapshot-upload-delay=2s # The OTLP endpoint comes from the ate-otel-config ConfigMap the # atelet DaemonSet consumes via envFrom; the kind copy lives at # ../ate-otel-config.yaml and is applied by the parent overlay or From c093a40cbb8ac2cfd61ec075ac064e8aa35c0633 Mon Sep 17 00:00:00 2001 From: Julian Gutierrez Oschmann Date: Wed, 28 Oct 2026 00:34:24 +0000 Subject: [PATCH 22/53] Add kubectl ate proxy actor command. (#3191) Allow a user to land traffic on an actor from a dev machine. The `kubectl-ate` CLI now authenticates to `atenet` by minting a certificate (via `PodCertificateRequest`) for `spiffe://cluster.local/ns/ate-system/sa/atenet-client`. Ideally, atenet should support JWT based auth and there would be no need for a certificate minting. But that support is not in place yet. The `kubectl ate proxy actor` command also improves local dev ergonomics as it attachjes the actor header so clients don't need to be "substrate-aware". For example, a web browser can just point to the local proxy. Fixed #3194 and #3187 --------- Signed-off-by: Julian Gutierrez Oschmann --- README.md | 28 +- cmd/ate-setup/commands.md | 2 +- cmd/ate-setup/internal/config/config.go | 13 +- cmd/ate-setup/internal/config/config_test.go | 28 -- cmd/ate-setup/internal/config/setting.go | 4 +- cmd/ate-setup/internal/steps/deploy.go | 9 +- cmd/ate-setup/internal/steps/overlay.go | 15 +- cmd/ate-setup/internal/steps/overlay_test.go | 31 +- cmd/atenet/internal/router/README.md | 28 +- cmd/atenet/internal/router/cmd.go | 2 +- cmd/atenet/internal/router/config.go | 5 +- cmd/atenet/internal/router/config_test.go | 5 +- cmd/atenet/internal/router/dataplane.go | 10 +- cmd/atenet/internal/router/xds.go | 22 +- cmd/atenet/internal/router/xds_test.go | 26 +- cmd/kubectl-ate/README.md | 15 + cmd/kubectl-ate/internal/cmd/proxy.go | 240 ++++++++++++++ cmd/kubectl-ate/internal/cmd/proxy_test.go | 204 ++++++++++++ .../internal/cmd/validation_test.go | 18 ++ cmd/kubectl-ate/internal/cmd/verbs.go | 2 +- demos/counter/README.md | 73 ++--- demos/egress/README.md | 15 +- demos/egress/test-egress.sh | 8 +- demos/jupyter/README.md | 26 +- demos/jupyter/jupyter.yaml.tmpl | 70 ----- demos/multi-template/README.md | 21 +- demos/parking/README.md | 20 +- demos/parking/load.sh | 55 ++-- demos/sandbox/README.md | 25 +- demos/sandbox/client/main.go | 27 +- docs/api-guide.md | 32 +- docs/operator-install.md | 13 + hack/run-microvm-demo.sh | 11 +- hack/verify-egress-demo.sh | 6 +- internal/ateclient/builder.go | 10 +- internal/ateclient/builder_test.go | 4 +- internal/ateclient/router.go | 266 ++++++++++++++++ internal/ateclient/router_test.go | 294 ++++++++++++++++++ internal/e2e/ingressauth.go | 196 +----------- internal/e2e/router_client.go | 80 +---- internal/e2e/router_client_test.go | 12 +- internal/podidentity/podidentity.go | 216 +++++++++++++ .../podidentity_test.go} | 60 ++-- manifests/ate-install/atenet-router.yaml | 6 +- 44 files changed, 1569 insertions(+), 684 deletions(-) create mode 100644 cmd/kubectl-ate/internal/cmd/proxy.go create mode 100644 cmd/kubectl-ate/internal/cmd/proxy_test.go create mode 100644 internal/ateclient/router.go create mode 100644 internal/ateclient/router_test.go create mode 100644 internal/podidentity/podidentity.go rename internal/{e2e/ingressauth_test.go => podidentity/podidentity_test.go} (76%) diff --git a/README.md b/README.md index a6e0f654b4..702da9c8ba 100644 --- a/README.md +++ b/README.md @@ -115,14 +115,15 @@ To quickly set up the complete environment: 2. Run the following steps: ```shell -# create cluster and local registry (IPv4; IP_FAMILY=dual|ipv6 overrides) +# create cluster and local registry hack/create-kind-cluster.sh -# install ate, PostgreSQL, rustfs -hack/install-ate-kind.sh deploy ate-system --credential-provider='{"name":"k8s.io"}' +# install Substrate +hack/install-ate-kind.sh deploy ate-system --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-system/sa/atenet-client # install counter demo -hack/install-ate-kind.sh deploy demo-counter +hack/install-ate-kind.sh deploy demo counter # install kubectl-ate go install ./cmd/kubectl-ate @@ -131,8 +132,9 @@ go install ./cmd/kubectl-ate # actor template, resolved in the actor's atespace) kubectl ate create actor my-counter-1 -a ate-demo-counter --template counter -# port-forward the network router to bind to local port `8000` -kubectl port-forward -n ate-system svc/atenet-router 8000:80 +# serve a local proxy on port 8000 that sends requests to the actor through the +# network router +kubectl ate proxy actor my-counter-1 -a ate-demo-counter ``` > [!IMPORTANT] @@ -143,9 +145,7 @@ kubectl port-forward -n ate-system svc/atenet-router 8000:80 3. In a **separate terminal**, send an HTTP request to increment the counter: ```shell -curl -X POST \ - -H "ate-target-actor: ate-demo-counter/my-counter-1" \ - -i http://localhost:8000/ +curl -X POST -i http://localhost:8000/ ``` ### GKE Quickstart (Development) @@ -179,9 +179,15 @@ curl -X POST \ 4. Deploy the Agent Substrate system to your cluster: ```bash - go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}' + go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-system/sa/atenet-client ``` + `--ingress-allowed-spiffe-ids` lists the clients the router lets reach + actors. The ID above is the one `kubectl ate proxy actor` presents. Without + the flag no client reaches actors. See + [Ingress to actors](docs/operator-install.md#ingress-to-actors). + `--credential-provider` is required. `{"name":"k8s.io"}` also deploys the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off @@ -195,7 +201,7 @@ curl -X POST \ 5. You can then deploy the sample applications. See [demos/counter/README.md](demos/counter/README.md) or [demos/sandbox/README.md](demos/sandbox/README.md) for detailed walkthroughs. ```bash - go run ./cmd/ate-setup deploy demo demo-counter + go run ./cmd/ate-setup deploy demo counter ``` #### Custom Setup and Deployment diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 60665e3ae9..b10ce64d40 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -31,7 +31,7 @@ a pre-scan pass, so they may appear anywhere on its command line. | `--ateapi-postgres-storage-class NAME` | `ATE_API_POSTGRES_STORAGE_CLASS=NAME` | StorageClass for the bundled PostgreSQL volume instead of the cluster default, such as `dynamic-rwo` on GKE so C4 and N4 nodes can attach it. An existing install must delete the postgres StatefulSet and its claim first | | `--cordon-system-infra-components` | `--cordon-system-infra-components` | Keep the system infra components, such as ate-api-server, atenet-router, atenet-egress and postgres, off the worker nodes. Assumes a small shared pool labeled and tainted `ate.dev/workloadType=ate-system-infra:NoSchedule`, across which each workload's replicas are spread, and a one-node pool labeled and tainted `ate.dev/workloadType=ate-postgres:NoSchedule` for postgres alone. `ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS=true` when the flag is absent | | `--fake-data-plane` | `--fake-data-plane` | Install the fake data plane for control-plane benchmarks: fake-atelet takes atelet's place in its DaemonSet, taking its objectstore-plugin's storage settings, ate-controller is scaled to zero, and fake-workersync honors every WorkerPool with fake Workers on the fake-atelet nodes. Reaches `deploy ate-system`, `deploy atelet`, and `deploy ate-controller`; only `deploy ate-system` switches a cluster between the real and fake data planes, and the other two refuse a data plane other than the installed one. `deploy ate-system --fake-data-plane` refuses a cluster whose WorkerPools have worker pods. fake-atelet runs with its defaults; a benchmark tunes it through the optional `fake-atelet-tuning` ConfigMap, whose `FAKE_ATELET_*` variables set its flags, and which no install creates or resets. Actors do not exist; never use it on a cluster that serves real actors. `ATE_INSTALL_FAKE_DATA_PLANE=true` when the flag is absent | -| `--ingress-allowed-spiffe-ids IDS` | `--ingress-allowed-spiffe-ids IDS`, or `ATE_INGRESS_ALLOWED_SPIFFE_IDS=IDS` | Required by `deploy ate-system` and `deploy atenet` for both dataplanes. Comma-separated SPIFFE IDs (exact match) of the clients the ingress router lets reach actors; any listed client may reach any actor. Both dataplanes require a Pod Identity CA client certificate on HTTPS and TLS CONNECT. The SPIFFE URI SAN allowlist may be checked during or after TLS, including after CONNECT acceptance, but unlisted clients must be denied before actor routing. The podidentity signer issues `spiffe://cluster.local/ns//sa/` | +| `--ingress-allowed-spiffe-ids IDS` | `--ingress-allowed-spiffe-ids IDS`, or `ATE_INGRESS_ALLOWED_SPIFFE_IDS=IDS` | Comma-separated SPIFFE IDs (exact match) of the clients the ingress router lets reach actors; any listed client may reach any actor. Unset, no client may reach actors through the router. Both dataplanes require a Pod Identity CA client certificate on HTTPS and TLS CONNECT. The SPIFFE URI SAN allowlist may be checked during or after TLS, including after CONNECT acceptance, but unlisted clients must be denied before actor routing. The podidentity signer issues `spiffe://cluster.local/ns//sa/` | | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | | `--credential-provider JSON` | `--credential-provider JSON`, or `ATE_CREDENTIAL_PROVIDER` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/egress-credential-injection.md`](../../docs/egress-credential-injection.md) | | `--ateapi-log-level LEVEL` | `--ateapi-log-level LEVEL`, or `ATE_API_LOG_LEVEL=LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). The server picks up a change without restarting | diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index e5de83a236..7a80966d89 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -223,7 +223,7 @@ type Config struct { // IngressAllowedSPIFFEIDs are the SPIFFE IDs the ingress router lets // reach actors over its TLS listeners (ATE_INGRESS_ALLOWED_SPIFFE_IDS). - // See RequireIngressAllowedSPIFFEIDs. + // Empty means no client may reach actors through the router. IngressAllowedSPIFFEIDs []string // AdditionalEgressExtprocService is the optional NS/SVC:PORT external processor filter. @@ -282,17 +282,6 @@ func (c *Config) Resolved() *Resolved { return c.resolved } -// RequireIngressAllowedSPIFFEIDs fails when the install would deploy an -// ingress router without an allowlist. Envoy and -// AgentGateway both authenticate clients on their TLS listeners and use the -// allowlist to decide which authenticated identities may reach actors. -func (c *Config) RequireIngressAllowedSPIFFEIDs() error { - if len(c.IngressAllowedSPIFFEIDs) > 0 { - return nil - } - return c.Resolved().Require("ingressAllowedSPIFFEIDs") -} - // TransferServiceAccount is the email of the snapshot transfer service // account, built from TransferGSAName and ProjectID as tools/setup-gcp builds // it. It is empty when either is. diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 58f87c9338..b50366e1d9 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -316,34 +316,6 @@ func TestLoadIngressAllowedSPIFFEIDs(t *testing.T) { } } -// Both ingress dataplanes require an allowlist. -func TestRequireIngressAllowedSPIFFEIDs(t *testing.T) { - for _, tc := range []struct { - name string - cfg Config - wantErr bool - }{ - {name: "envoy without IDs", cfg: Config{Router: RouterEnvoy}, wantErr: true}, - {name: "envoy with IDs", cfg: Config{Router: RouterEnvoy, IngressAllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/a/sa/a"}}}, - {name: "agentgateway without IDs", cfg: Config{Router: RouterAgentgateway}, wantErr: true}, - {name: "agentgateway with IDs", cfg: Config{Router: RouterAgentgateway, IngressAllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/a/sa/a"}}}, - } { - t.Run(tc.name, func(t *testing.T) { - err := tc.cfg.RequireIngressAllowedSPIFFEIDs() - if !tc.wantErr { - if err != nil { - t.Fatalf("RequireIngressAllowedSPIFFEIDs() = %v, want nil", err) - } - return - } - var required *RequiredError - if !errors.As(err, &required) || required.Setting.Key != "ingressAllowedSPIFFEIDs" { - t.Fatalf("RequireIngressAllowedSPIFFEIDs() = %v, want a RequiredError for ingressAllowedSPIFFEIDs", err) - } - }) - } -} - // The credential exchange impersonates the transfer account only on GCS, so // only a GKE deploy needs one, and its email needs both the name and the // project. diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index aa2b12181c..1fdea0effb 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -311,8 +311,8 @@ var Registry = []Setting{ Key: "ingressAllowedSPIFFEIDs", Env: "ATE_INGRESS_ALLOWED_SPIFFE_IDS", Flag: "ingress-allowed-spiffe-ids", Kind: KindString, Usage: "SPIFFE IDs (exact match, comma-separated) of the clients the ingress router lets reach actors; " + - "any listed client may reach any actor. Required to deploy the router with " + - "either dataplane. The podidentity signer issues spiffe://cluster.local/ns//sa/", + "any listed client may reach any actor. Unset, no client may reach actors through the router. " + + "The podidentity signer issues spiffe://cluster.local/ns//sa/", }, { Key: "otlpEndpoint", Env: "ATE_OTLP_ENDPOINT", Flag: "otlp-endpoint", Kind: KindString, diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index e235e96b0a..bacf30a4e5 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -68,11 +68,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err != nil { return err } - // Likewise a router that would refuse to start without an allowlist, and - // a credential exchange without a transfer service account. - if err := e.Cfg.RequireIngressAllowedSPIFFEIDs(); err != nil { - return err - } + // Likewise a credential exchange without a transfer service account. if err := e.Cfg.RequireTransferServiceAccount(); err != nil { return err } @@ -538,9 +534,6 @@ func (e *Env) DeployAtenet(ctx context.Context) error { if err != nil { return err } - if err := e.Cfg.RequireIngressAllowedSPIFFEIDs(); err != nil { - return err - } if err := e.EnsureCRDs(ctx); err != nil { return err } diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 78ef2f4596..401b5b2b0d 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -206,11 +206,12 @@ func (e *Env) setAgentgatewayIngressAllowedSPIFFEIDs(raw []byte) ([]byte, error) if e.Cfg.Router != config.RouterAgentgateway { return raw, nil } - if err := e.Cfg.RequireIngressAllowedSPIFFEIDs(); err != nil { - return nil, err + // nil would marshal as null, which agentgateway's networkAuthorization CEL rule can't evaluate. + allowed := e.Cfg.IngressAllowedSPIFFEIDs + if allowed == nil { + allowed = []string{} } - - ids, err := json.Marshal(e.Cfg.IngressAllowedSPIFFEIDs) + ids, err := json.Marshal(allowed) if err != nil { return nil, fmt.Errorf("encoding AgentGateway ingress SPIFFE IDs: %w", err) } @@ -297,10 +298,10 @@ func patchAtenetEgressEnvoyConcurrency(raw []byte, concurrency int) ([]byte, err // patchAtenetRouterAllowedSPIFFEIDs appends --ingress-allowed-spiffe-ids to the // atenet-router container's args in the Envoy router Deployment. The base // manifest carries no allowlist, because which clients may reach actors is the -// install's to decide. +// install's to decide. Without one the router serves no traffic. func (e *Env) patchAtenetRouterAllowedSPIFFEIDs(raw []byte) ([]byte, error) { - if err := e.Cfg.RequireIngressAllowedSPIFFEIDs(); err != nil { - return nil, err + if len(e.Cfg.IngressAllowedSPIFFEIDs) == 0 { + return raw, nil } return appendContainerArgs(raw, "atenet-router", "atenet-router", "--ingress-allowed-spiffe-ids="+strings.Join(e.Cfg.IngressAllowedSPIFFEIDs, ",")) diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 066cbafb23..7de3d99195 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -16,7 +16,6 @@ package steps import ( "context" - "errors" "fmt" "os" "reflect" @@ -887,7 +886,7 @@ func TestRenderE2ETuning(t *testing.T) { } // Both paths that deliver the Envoy router pass it the install's allowlist, -// whatever the overlay, and refuse to render without one. +// whatever the overlay, and pass none when the install sets none. func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { const a, b = "spiffe://cluster.local/ns/a/sa/a", "spiffe://cluster.local/ns/b/sa/b" src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} @@ -938,10 +937,14 @@ func TestRenderRouterAllowedSPIFFEIDs(t *testing.T) { t.Run("without IDs", func(t *testing.T) { cfg := cfg cfg.IngressAllowedSPIFFEIDs = nil - _, err := tc.render(&Env{Cfg: &cfg, resolver: resolver}) - var required *config.RequiredError - if !errors.As(err, &required) { - t.Fatalf("render error = %v, want a RequiredError", err) + rendered, err := tc.render(&Env{Cfg: &cfg, resolver: resolver}) + if err != nil { + t.Fatalf("render: %v", err) + } + for _, arg := range routerArgs(t, rendered) { + if strings.HasPrefix(arg, "--ingress-allowed-spiffe-ids") { + t.Errorf("atenet-router args include %q, want no allowlist", arg) + } } }) }) @@ -1066,10 +1069,18 @@ func TestRenderAgentgatewayStaticMTLS(t *testing.T) { t.Run("without IDs", func(t *testing.T) { cfg.IngressAllowedSPIFFEIDs = nil - _, err := tc.render(e) - var required *config.RequiredError - if !errors.As(err, &required) { - t.Fatalf("render error = %v, want a RequiredError", err) + manifest, err := tc.render(e) + if err != nil { + t.Fatalf("rendering static-mtls AgentGateway manifests: %v", err) + } + var gateway map[string]interface{} + if err := yaml.Unmarshal([]byte(agentgatewayRouterConfig(t, manifest).Data["config.yaml"]), &gateway); err != nil { + t.Fatalf("parsing rendered AgentGateway config: %v", err) + } + authorization := gateway["frontendPolicies"].(map[string]interface{})["networkAuthorization"].(map[string]interface{}) + wantRules := []interface{}{map[string]interface{}{"allow": "source.spiffeId in []"}} + if got := authorization["rules"]; !reflect.DeepEqual(got, wantRules) { + t.Errorf("network authorization rules = %v, want %v", got, wantRules) } }) }) diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index 2d7076bfa7..06777c6db6 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -216,7 +216,8 @@ Right now the only "authorization" policy is to check the client's SPIFFE ID against a static allowlist. Any ID on the allowlist may use the ingress router to contact any actor. Richer authorization policies may be added post-GA. Use the `--ingress-allowed-spiffe-ids` flag to set the allowlist (comma-separated -list). +list). With an empty allowlist, the Envoy router serves no listener, so no +traffic reaches actors through it. ### Common setups @@ -257,15 +258,22 @@ volumes: Add the pod's SPIFFE ID to `--ingress-allowed-spiffe-ids`. -**Client outside the cluster, pod identity CA.** A demo on your machine can -follow the e2e tests in `internal/e2e/ingressauth.go`. It creates a -PodCertificateRequest for the podidentity signer that names a pod, node and -ServiceAccount that don't exist. kube-apiserver checks those only for requests -from a kubelet, and the signer issues the SPIFFE ID of the named namespace and -ServiceAccount. The client keeps the private key, reads the chain from the -request's status, and reads the servicedns ClusterTrustBundle to verify the -router. Creating PodCertificateRequests needs RBAC for that resource in the -namespace. +**Client outside the cluster, pod identity CA.** `kubectl ate proxy actor` +forwards local TCP ports to an actor through the router's CONNECT listener. It +presents a certificate for +`spiffe://cluster.local/ns/ate-system/sa/atenet-client`, so that ID must be in +`--ingress-allowed-spiffe-ids`. It requests the certificate the way +`internal/podidentity` does: a PodCertificateRequest for the podidentity +signer that names a pod, node and ServiceAccount that don't exist. +kube-apiserver checks those only for requests from a kubelet, and the signer +issues the SPIFFE ID of the named namespace and ServiceAccount. The client +keeps the private key, reads the chain from the request's status, and reads +the servicedns ClusterTrustBundle to verify the router. + +Creating PodCertificateRequests needs RBAC for that resource in `ate-system`. +That RBAC can request a certificate for any SPIFFE ID in `ate-system`, +including those of atelet, ate-controller and ate-api-server, so grant it only +to users you would trust with those components. ## status page diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index 17620d57fe..42ccd65da9 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -72,7 +72,7 @@ func bindRouterFlags(fs *pflag.FlagSet, cfg *routerConfig) { // Flags for client authentication and authorization in ingress mode. fs.StringVar(&cfg.IngressAuth.ClientCAFile, "ingress-client-ca-file", "", "PEM bundle of the CAs allowed to sign ingress client certificates. Envoy reads the file, so it must be mounted at this path in the Envoy container as well. Required for the Envoy ingress dataplane") - fs.StringSliceVar(&cfg.IngressAuth.AllowedSPIFFEIDs, "ingress-allowed-spiffe-ids", nil, "SPIFFE IDs (exact match) allowed to connect to actors through the ingress dataplane. Comma-separated or repeated. Required for the Envoy ingress dataplane") + fs.StringSliceVar(&cfg.IngressAuth.AllowedSPIFFEIDs, "ingress-allowed-spiffe-ids", nil, "SPIFFE IDs (exact match) allowed to connect to actors through the ingress dataplane. Comma-separated or repeated. Without any, the Envoy ingress dataplane serves no traffic") fs.StringVar(&cfg.UpstreamCredentialBundlePath, "upstream-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle (cert+key) the router presents as the client cert when dialing the actor's atunnel ingress server over mTLS. Empty disables upstream mTLS (legacy plaintext pod-IP:80) and needs --upstream-connect=false.") fs.StringVar(&cfg.UpstreamTrustBundlePath, "upstream-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to validate the actor's atunnel ingress server certificate.") diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 5345caef7d..8d3fe2cfde 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -73,7 +73,7 @@ type ingressAuthConfig struct { // container at the same path. ClientCAFile string // AllowedSPIFFEIDs is the exact set of SPIFFE IDs allowed to connect to - // actors. + // actors. When it is empty, the dataplane serves no traffic. // // Entries are complete SPIFFE IDs, like // `spiffe://mytrustdomain.com/foo/bar/baz`. @@ -326,9 +326,6 @@ func (c routerConfig) validateIngressAuth() error { if c.IngressAuth.ClientCAFile == "" { return fmt.Errorf("--ingress-client-ca-file is required") } - if len(c.IngressAuth.AllowedSPIFFEIDs) == 0 { - return fmt.Errorf("at least one --ingress-allowed-spiffe-ids entry is required") - } for _, id := range c.IngressAuth.AllowedSPIFFEIDs { if _, err := spiffeid.FromString(id); err != nil { return fmt.Errorf("--ingress-allowed-spiffe-ids: %q is not a valid SPIFFE ID: %w", id, err) diff --git a/cmd/atenet/internal/router/config_test.go b/cmd/atenet/internal/router/config_test.go index d047cee62a..43744e6e04 100644 --- a/cmd/atenet/internal/router/config_test.go +++ b/cmd/atenet/internal/router/config_test.go @@ -219,9 +219,8 @@ func TestRouterConfigValidateIngressAuth(t *testing.T) { wantErr: "--ingress-client-ca-file is required", }, { - name: "empty allowlist rejected", - mutate: func(c *routerConfig) { c.IngressAuth.AllowedSPIFFEIDs = nil }, - wantErr: "at least one --ingress-allowed-spiffe-ids entry is required", + name: "empty allowlist accepted", + mutate: func(c *routerConfig) { c.IngressAuth.AllowedSPIFFEIDs = nil }, }, { name: "non-SPIFFE ID rejected", diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index 8e455ff8f1..3ce608ce32 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -109,9 +109,13 @@ func (s *RouterServer) newEnvoyXdsServer(ctx context.Context, parkCfg ingress.Pa return nil, fmt.Errorf("loading --ingress-client-ca-file: %w", err) } xdsSrv.SetDownstreamClientAuth(s.cfg.IngressAuth.ClientCAFile, s.cfg.IngressAuth.AllowedSPIFFEIDs) - slog.InfoContext(ctx, "TLS ingress listeners require client certificates", - slog.String("client_ca_file", s.cfg.IngressAuth.ClientCAFile), - slog.Any("allowed_spiffe_ids", s.cfg.IngressAuth.AllowedSPIFFEIDs)) + if len(s.cfg.IngressAuth.AllowedSPIFFEIDs) == 0 { + slog.InfoContext(ctx, "No --ingress-allowed-spiffe-ids, so the router serves no traffic") + } else { + slog.InfoContext(ctx, "TLS ingress listeners require client certificates", + slog.String("client_ca_file", s.cfg.IngressAuth.ClientCAFile), + slog.Any("allowed_spiffe_ids", s.cfg.IngressAuth.AllowedSPIFFEIDs)) + } xdsSrv.SetUpstreamTls(s.cfg.UpstreamCredentialBundlePath, s.cfg.UpstreamTrustBundlePath) xdsSrv.SetUpstreamConnect(s.cfg.UpstreamConnect) diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index 8f9e798eda..a72ad275fa 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -385,8 +385,8 @@ func (x *XdsServer) SetTlsConfig(httpsPort int, certPath string) { // SetDownstreamClientAuth makes every TLS ingress listener require a client // certificate signed by a CA in caPath whose URI SAN exactly matches one of // allowedSPIFFEIDs. Envoy enforces this during the handshake, so a rejected -// client never reaches ext_proc. UpdateSnapshot fails if a TLS listener is -// enabled without it. +// client never reaches ext_proc. UpdateSnapshot serves no TLS listener when +// allowedSPIFFEIDs is empty, and fails if one is enabled without caPath. func (x *XdsServer) SetDownstreamClientAuth(caPath string, allowedSPIFFEIDs []string) { x.mu.Lock() defer x.mu.Unlock() @@ -520,12 +520,15 @@ func (x *XdsServer) UpdateSnapshot() error { x.mu.Lock() defer x.mu.Unlock() - httpsEnabled := x.httpsPort > 0 && x.certPath != "" + // Without an allowlist no client may reach actors, so the TLS listeners + // are not served. Envoy skips the SAN check when there are no matchers. + clientsAllowed := len(x.downstreamAllowedSPIFFEIDs) > 0 + httpsEnabled := x.httpsPort > 0 && x.certPath != "" && clientsAllowed // The main_internal cluster and listener only exist to serve the CONNECT // listener. - connectEnabled := x.connectTLSPort > 0 && x.certPath != "" - if (httpsEnabled || connectEnabled) && (x.downstreamClientCAPath == "" || len(x.downstreamAllowedSPIFFEIDs) == 0) { - return fmt.Errorf("TLS ingress listeners need client authentication: call SetDownstreamClientAuth") + connectEnabled := x.connectTLSPort > 0 && x.certPath != "" && clientsAllowed + if (httpsEnabled || connectEnabled) && x.downstreamClientCAPath == "" { + return fmt.Errorf("TLS ingress listeners need a client CA: call SetDownstreamClientAuth") } x.versionCount++ @@ -551,9 +554,10 @@ func (x *XdsServer) UpdateSnapshot() error { clusters = append(clusters, x.buildOtlpCollectorCluster()) } - // Routes - routes := []types.Resource{ - x.buildRoutes(), + // Routes. Only the HTTPS and main_internal listeners reference them. + var routes []types.Resource + if httpsEnabled || connectEnabled { + routes = append(routes, x.buildRoutes()) } // Listeners diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 1bcfec6a3d..e616ed70b4 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -1652,23 +1652,39 @@ func TestXdsServer_UpdateSnapshot_DownstreamClientAuth(t *testing.T) { } // A TLS listener without client authentication would let any client reach any -// actor, so UpdateSnapshot refuses to build one. +// actor. With no allowlist UpdateSnapshot serves no TLS listener, and with an +// allowlist but no CA it refuses to build one. func TestXdsServer_UpdateSnapshot_TLSListenerWithoutClientAuth(t *testing.T) { for _, tc := range []struct { name string httpsPort int connectTLSPort int + caPath string + allowed []string + wantErr bool }{ - {name: "HTTPS", httpsPort: 8443}, - {name: "CONNECT-TLS", connectTLSPort: 8444}, + {name: "HTTPS without allowlist", httpsPort: 8443, caPath: "/run/podidentity.podcert.ate.dev/trust-bundle.pem"}, + {name: "CONNECT-TLS without allowlist", connectTLSPort: 8444, caPath: "/run/podidentity.podcert.ate.dev/trust-bundle.pem"}, + {name: "HTTPS without CA", httpsPort: 8443, allowed: []string{"spiffe://cluster.local/ns/demo/sa/client"}, wantErr: true}, + {name: "CONNECT-TLS without CA", connectTLSPort: 8444, allowed: []string{"spiffe://cluster.local/ns/demo/sa/client"}, wantErr: true}, } { t.Run(tc.name, func(t *testing.T) { server := NewXdsServer(18000) server.SetConfig(50053, "127.0.0.1") server.SetConnectTLSPort(tc.connectTLSPort) server.SetTlsConfig(tc.httpsPort, "/run/servicedns.podcert.ate.dev/credential-bundle.pem") - if err := server.UpdateSnapshot(); err == nil { - t.Fatal("UpdateSnapshot succeeded, want an error") + server.SetDownstreamClientAuth(tc.caPath, tc.allowed) + if tc.wantErr { + if err := server.UpdateSnapshot(); err == nil { + t.Fatal("UpdateSnapshot succeeded, want an error") + } + return + } + listeners := snapshotListeners(t, server) + for _, name := range []string{IngressHTTPSListener, "connect_terminate_tls"} { + if _, exists := listeners[name]; exists { + t.Errorf("TLS listener '%s' is served without an allowlist", name) + } } }) } diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index d0ac8f2528..340d696e26 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -222,6 +222,21 @@ kubectl ate revert actor my-actor -a kubectl ate delete actor my-actor -a ``` +### Sending Requests to an Actor + +`proxy actor` forwards one or more local TCP ports to an actor. It presents +the client certificate for `spiffe://cluster.local/ns/ate-system/sa/atenet-client`, so the install must list that ID in +`--ingress-allowed-spiffe-ids`. + +```bash +# Forward 127.0.0.1:8000 to actor port 80 (default when no ports are given). +kubectl ate proxy actor my-actor -a +curl http://127.0.0.1:8000/ + +# Forward specific [local-port:]remote-port mappings. +kubectl ate proxy actor my-actor 8000:80 9090:9090 -a +``` + ### Actor Snapshots Suspending an actor creates a durable snapshot. Tags give snapshots stable, diff --git a/cmd/kubectl-ate/internal/cmd/proxy.go b/cmd/kubectl-ate/internal/cmd/proxy.go new file mode 100644 index 0000000000..57ae977d63 --- /dev/null +++ b/cmd/kubectl-ate/internal/cmd/proxy.go @@ -0,0 +1,240 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strconv" + "strings" + "sync" + + "github.com/spf13/cobra" + + "go.ate.dev/substrate/internal/ateclient" + "go.ate.dev/substrate/internal/resources" +) + +const ( + defaultProxyLocalPort = 8000 + defaultActorPort = 80 +) + +var ( + proxyActorAtespaceFlag string + proxyActorAddressFlag string +) + +type portMapping struct { + localPort int + remotePort int +} + +type actorDialFunc func(ctx context.Context, actorRef resources.ActorRef, port int) (net.Conn, error) + +var proxyCmd = &cobra.Command{Use: "proxy", Short: "Serve a local proxy to a resource"} + +var proxyActorCmd = &cobra.Command{ + Use: "actor [[local-port:]remote-port ...]", + Short: "Forward one or more local ports to an actor", + Long: `Forward one or more local TCP ports to an actor through Substrate's router, +resuming the actor if it is suspended. When no ports are +given, local port 8000 is forwarded to actor port 80. + +Requires permission to create, get, and delete podcertificaterequests in +Substrate's system namespace.`, + Args: func(cmd *cobra.Command, args []string) error { + if err := checkSwallowedFlags(cmd); err != nil { + return err + } + missingFlags := missingRequiredFlags(cmd, []string{"atespace"}) + if len(args) == 0 { + if len(missingFlags) > 0 { + return usageErrorf("missing required argument and %s", formatMissingFlagsClause(missingFlags)) + } + return usageErrorf("missing required argument ") + } + if len(missingFlags) > 0 { + return usageErrorf("missing %s", formatMissingFlagsClause(missingFlags)) + } + _, err := parsePortMappings(args[1:]) + return err + }, + RunE: func(cmd *cobra.Command, args []string) error { + actorRef := resources.ActorRef{Atespace: proxyActorAtespaceFlag, Name: args[0]} + mappings, err := parsePortMappings(args[1:]) + if err != nil { + return err + } + // TODO: Authenticate with a JWT once atenet-router supports JWT-based + // authentication instead of minting a client certificate via PodCertificateRequest. + dialer, err := ateclient.NewRouterDialer(cmd.Context(), kubeconfig, k8sContext) + if err != nil { + return fmt.Errorf("failed to connect to atenet-router: %w", err) + } + listeners := make([]net.Listener, 0, len(mappings)) + defer func() { + for _, lis := range listeners { + _ = lis.Close() + } + }() + for _, m := range mappings { + lis, err := net.Listen("tcp", net.JoinHostPort(proxyActorAddressFlag, strconv.Itoa(m.localPort))) + if err != nil { + return err + } + listeners = append(listeners, lis) + fmt.Fprintf(cmd.ErrOrStderr(), "Proxying %s -> %s:%d\n", lis.Addr(), actorRef, m.remotePort) + } + return serveActorProxy(cmd.Context(), cmd.ErrOrStderr(), listeners, mappings, actorRef, dialer.DialActor) + }, +} + +func parsePortMappings(specs []string) ([]portMapping, error) { + if len(specs) == 0 { + return []portMapping{{localPort: defaultProxyLocalPort, remotePort: defaultActorPort}}, nil + } + mappings := make([]portMapping, 0, len(specs)) + seenLocal := make(map[int]bool, len(specs)) + for _, spec := range specs { + localStr, remoteStr, hasColon := strings.Cut(spec, ":") + if !hasColon { + port, err := parsePortNumber(spec, 1) + if err != nil { + return nil, usageErrorf("invalid port %q: must be 1-65535 or [local-port:]remote-port", spec) + } + if seenLocal[port] { + return nil, usageErrorf("duplicate local port %d", port) + } + seenLocal[port] = true + mappings = append(mappings, portMapping{localPort: port, remotePort: port}) + continue + } + remotePort, err := parsePortNumber(remoteStr, 1) + if err != nil { + return nil, usageErrorf("invalid remote port in %q: must be 1-65535", spec) + } + var localPort int + if localStr != "" { + localPort, err = parsePortNumber(localStr, 0) + if err != nil { + return nil, usageErrorf("invalid local port in %q: must be 0-65535", spec) + } + } + if localPort != 0 { + if seenLocal[localPort] { + return nil, usageErrorf("duplicate local port %d", localPort) + } + seenLocal[localPort] = true + } + mappings = append(mappings, portMapping{localPort: localPort, remotePort: remotePort}) + } + return mappings, nil +} + +func parsePortNumber(s string, minPort int) (int, error) { + p, err := strconv.Atoi(s) + if err != nil || p < minPort || p > 65535 { + return 0, fmt.Errorf("invalid port %q", s) + } + return p, nil +} + +func serveActorProxy(ctx context.Context, errOut io.Writer, listeners []net.Listener, mappings []portMapping, actorRef resources.ActorRef, dial actorDialFunc) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + stopAfter := context.AfterFunc(ctx, func() { + for _, lis := range listeners { + _ = lis.Close() + } + }) + defer stopAfter() + + errCh := make(chan error, len(listeners)) + var wg sync.WaitGroup + for i, lis := range listeners { + remotePort := mappings[i].remotePort + wg.Add(1) + go func(lis net.Listener, remotePort int) { + defer wg.Done() + for { + clientConn, err := lis.Accept() + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + errCh <- nil + return + } + errCh <- err + cancel() + return + } + wg.Add(1) + go func(clientConn net.Conn) { + defer wg.Done() + handleActorConn(ctx, errOut, clientConn, actorRef, remotePort, dial) + }(clientConn) + } + }(lis, remotePort) + } + + var firstErr error + for range listeners { + if err := <-errCh; err != nil && firstErr == nil { + firstErr = err + } + } + wg.Wait() + return firstErr +} + +func handleActorConn(ctx context.Context, errOut io.Writer, clientConn net.Conn, actorRef resources.ActorRef, remotePort int, dial actorDialFunc) { + defer clientConn.Close() + upstreamConn, err := dial(ctx, actorRef, remotePort) + if err != nil { + if ctx.Err() == nil && errOut != nil { + fmt.Fprintf(errOut, "Failed to connect to %s:%d: %v\n", actorRef, remotePort, err) + } + return + } + defer upstreamConn.Close() + + stopAfter := context.AfterFunc(ctx, func() { + _ = clientConn.Close() + _ = upstreamConn.Close() + }) + defer stopAfter() + + var wg sync.WaitGroup + wg.Add(2) + copyHalf := func(dst, src net.Conn) { + defer wg.Done() + _, _ = io.Copy(dst, src) + if cw, ok := dst.(interface{ CloseWrite() error }); ok { + _ = cw.CloseWrite() + } + } + go copyHalf(upstreamConn, clientConn) + go copyHalf(clientConn, upstreamConn) + wg.Wait() +} + +func init() { + proxyActorCmd.Flags().StringVarP(&proxyActorAtespaceFlag, "atespace", "a", "", "the actor's atespace") + proxyActorCmd.Flags().StringVar(&proxyActorAddressFlag, "address", "127.0.0.1", "the local address to listen on") + proxyCmd.AddCommand(proxyActorCmd) +} diff --git a/cmd/kubectl-ate/internal/cmd/proxy_test.go b/cmd/kubectl-ate/internal/cmd/proxy_test.go new file mode 100644 index 0000000000..74b6d4069c --- /dev/null +++ b/cmd/kubectl-ate/internal/cmd/proxy_test.go @@ -0,0 +1,204 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "fmt" + "io" + "net" + "slices" + "testing" + + "go.ate.dev/substrate/internal/resources" +) + +func TestParsePortMappings(t *testing.T) { + tests := []struct { + name string + specs []string + want []portMapping + wantErr bool + }{ + { + name: "default when omitted", + specs: nil, + want: []portMapping{{localPort: 8000, remotePort: 80}}, + }, + { + name: "single port maps local and remote to same port", + specs: []string{"9090"}, + want: []portMapping{{localPort: 9090, remotePort: 9090}}, + }, + { + name: "explicit local and remote port", + specs: []string{"8080:80"}, + want: []portMapping{{localPort: 8080, remotePort: 80}}, + }, + { + name: "random local port", + specs: []string{":80", "0:9090"}, + want: []portMapping{{localPort: 0, remotePort: 80}, {localPort: 0, remotePort: 9090}}, + }, + { + name: "multiple ports", + specs: []string{"8000:80", "9090"}, + want: []portMapping{{localPort: 8000, remotePort: 80}, {localPort: 9090, remotePort: 9090}}, + }, + { + name: "zero single port", + specs: []string{"0"}, + wantErr: true, + }, + { + name: "non-numeric port", + specs: []string{"http"}, + wantErr: true, + }, + { + name: "missing remote port", + specs: []string{"8000:"}, + wantErr: true, + }, + { + name: "zero remote port", + specs: []string{"8000:0"}, + wantErr: true, + }, + { + name: "remote port out of range", + specs: []string{"8000:65536"}, + wantErr: true, + }, + { + name: "negative local port", + specs: []string{"-1:80"}, + wantErr: true, + }, + { + name: "local port out of range", + specs: []string{"65536:80"}, + wantErr: true, + }, + { + name: "extra colon", + specs: []string{"8000:80:80"}, + wantErr: true, + }, + { + name: "duplicate local port", + specs: []string{"8000:80", "8000:9090"}, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := parsePortMappings(tc.specs) + if (err != nil) != tc.wantErr { + t.Fatalf("parsePortMappings(%v) error = %v, wantErr %t", tc.specs, err, tc.wantErr) + } + if !tc.wantErr && !slices.Equal(got, tc.want) { + t.Errorf("parsePortMappings(%v) = %v, want %v", tc.specs, got, tc.want) + } + }) + } +} + +func TestServeActorProxy(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + lis80, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer lis80.Close() + lis9090, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer lis9090.Close() + + wantActor := resources.ActorRef{Atespace: "team", Name: "my-counter"} + dial := func(_ context.Context, actorRef resources.ActorRef, port int) (net.Conn, error) { + if actorRef != wantActor { + return nil, fmt.Errorf("unexpected actor %s", actorRef) + } + backendLis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + go func() { + defer backendLis.Close() + serverSide, err := backendLis.Accept() + if err != nil { + return + } + defer serverSide.Close() + payload, err := io.ReadAll(serverSide) + if err != nil { + return + } + _, _ = fmt.Fprintf(serverSide, "%s:%d:%s", actorRef, port, payload) + }() + return net.Dial("tcp", backendLis.Addr().String()) + } + + done := make(chan error, 1) + go func() { + done <- serveActorProxy( + ctx, + io.Discard, + []net.Listener{lis80, lis9090}, + []portMapping{{remotePort: 80}, {remotePort: 9090}}, + wantActor, + dial, + ) + }() + + for _, tc := range []struct { + addr string + send string + want string + }{ + {addr: lis80.Addr().String(), send: "hello", want: "team/my-counter:80:hello"}, + {addr: lis9090.Addr().String(), send: "metrics", want: "team/my-counter:9090:metrics"}, + } { + conn, err := net.Dial("tcp", tc.addr) + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(conn, tc.send); err != nil { + t.Fatal(err) + } + if err := conn.(*net.TCPConn).CloseWrite(); err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(conn) + _ = conn.Close() + if err != nil { + t.Fatal(err) + } + if string(got) != tc.want { + t.Errorf("response from %s = %q, want %q", tc.addr, got, tc.want) + } + } + + cancel() + if err := <-done; err != nil { + t.Fatalf("serveActorProxy: %v", err) + } +} diff --git a/cmd/kubectl-ate/internal/cmd/validation_test.go b/cmd/kubectl-ate/internal/cmd/validation_test.go index 5e9ebbbe19..e49cde4f32 100644 --- a/cmd/kubectl-ate/internal/cmd/validation_test.go +++ b/cmd/kubectl-ate/internal/cmd/validation_test.go @@ -234,6 +234,24 @@ func TestCLIUsageErrorsGoToStderrWithUsage(t *testing.T) { wantErrorLine: "Error: missing required argument and required flag -a/--atespace", wantUsageLine: "kubectl-ate logs actor [flags]", }, + { + name: "proxy actor with nothing", + args: []string{"proxy", "actor"}, + wantErrorLine: "Error: missing required argument and required flag -a/--atespace", + wantUsageLine: "kubectl-ate proxy actor [[local-port:]remote-port ...] [flags]", + }, + { + name: "proxy actor missing -a", + args: []string{"proxy", "actor", "my-actor"}, + wantErrorLine: "Error: missing required flag -a/--atespace", + wantUsageLine: "kubectl-ate proxy actor [[local-port:]remote-port ...] [flags]", + }, + { + name: "proxy actor invalid port mapping", + args: []string{"proxy", "actor", "my-actor", "badport", "-a", "space-1"}, + wantErrorLine: `Error: invalid port "badport"`, + wantUsageLine: "kubectl-ate proxy actor [[local-port:]remote-port ...] [flags]", + }, { name: "create actor-template missing -f", args: []string{"create", "actor-template"}, diff --git a/cmd/kubectl-ate/internal/cmd/verbs.go b/cmd/kubectl-ate/internal/cmd/verbs.go index ebe169dbcf..f588d6f4bc 100644 --- a/cmd/kubectl-ate/internal/cmd/verbs.go +++ b/cmd/kubectl-ate/internal/cmd/verbs.go @@ -32,7 +32,7 @@ var ( ) func init() { - for _, c := range []*cobra.Command{getCmd, createCmd, updateCmd, deleteCmd, resumeCmd, suspendCmd, logsCmd, topCmd} { + for _, c := range []*cobra.Command{getCmd, createCmd, updateCmd, deleteCmd, resumeCmd, suspendCmd, logsCmd, topCmd, proxyCmd} { requireSubcommand(c) rootCmd.AddCommand(c) } diff --git a/demos/counter/README.md b/demos/counter/README.md index 98bb4b5329..4886c7d222 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -8,7 +8,8 @@ The demo uses two kinds of resources: the `WorkerPool` is a Kubernetes CRD, whil ## Prerequisites -- A k8s cluster with Agent Substrate installed. +- A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) + or [GKE Quickstart](../../README.md#gke-quickstart-development). - `ko` installed for building images. - A GCS bucket for storing snapshots (configured via `BUCKET_NAME` env var). @@ -114,45 +115,35 @@ kubectl ate create actor my-counter-1 -a ate-demo-counter --template counter ``` -### 3. Port-Forward Services +### 3. Proxy to the Actor -To interact with the router locally: +Start a local proxy that authenticates with `atenet-router` and forwards local port `8000` to port `80` on the actor: -```bash -# Port-forward the Atenet Router -kubectl port-forward -n ate-system svc/atenet-router 8000:80 +
+Proxy on MicroVM -# Also port-forward the CONNECT listener if you want to reach the actor's -# extra port (see "Reaching a non-default port" below). -kubectl port-forward -n ate-system svc/atenet-router 8001:8081 +```bash +kubectl ate proxy actor my-counter-microvm-1 -a ate-demo-counter-microvm ``` +
-## How to Use - -When you send an HTTP request through the router, Substrate automatically detects the session, activates (resumes) the actor onto an available worker pod, and proxies the traffic. -The two `-H` options below are the routing key; the request URL and `Host` are -ordinary application metadata. - -1. Send an HTTP POST request to increment the counter:
-Send to MicroVM +Proxy on GVisor ```bash -curl -X POST \ - -H "ate-target-actor: ate-demo-counter-microvm/my-counter-microvm-1" \ - http://localhost:8000 +kubectl ate proxy actor my-counter-1 -a ate-demo-counter ```
-
-Send to GVisor +## How to Use + +When you send an HTTP request through the proxy, `atenet-router` automatically activates (resumes) the actor onto an available worker pod and routes the traffic. + +1. In a separate terminal, send an HTTP POST request to increment the counter: ```bash -curl -X POST \ - -H "ate-target-actor: ate-demo-counter/my-counter-1" \ - http://localhost:8000 +curl -X POST http://localhost:8000 ``` -
2. Verify that the actor is now in a `RUNNING` state and assigned to a worker pod:
@@ -217,22 +208,14 @@ kubectl ate delete actor my-counter-1 -a ate-demo-counter `counter.go` also listens on a second, configurable port (`--extra-port=9090` in this demo's template) to demonstrate atenet-router's arbitrary-port ingress -support: a client reaches any port an actor listens on, not just its default -(port 80), by naming the port in an HTTP `CONNECT` request's authority rather -than a separate header. The router terminates the `CONNECT` on its own -listener (`--port-connect`, 8081 by default) and tunnels ordinary HTTP requests -through it to the named port. - -`curl -p` forces a tunnel even for a plain (non-TLS) target, which its default -proxy behavior wouldn't do: +support. Pass `[local-port:]remote-port` arguments to `kubectl ate proxy actor` +to forward additional ports:
Non standard port with MicroVM ```bash -curl -p -x http://localhost:8001 \ - --proxy-header "ate-target-actor: ate-demo-counter-microvm/my-counter-microvm-1" \ - http://my-counter-microvm-1:9090/ +kubectl ate proxy actor my-counter-microvm-1 8000:80 9090:9090 -a ate-demo-counter-microvm ```
@@ -240,16 +223,18 @@ curl -p -x http://localhost:8001 \ Non standard port with GVisor ```bash -curl -p -x http://localhost:8001 \ - --proxy-header "ate-target-actor: ate-demo-counter/my-counter-1" \ - http://my-counter-1:9090/ +kubectl ate proxy actor my-counter-1 8000:80 9090:9090 -a ate-demo-counter ```
-This reaches the same actor's second listener and resumes it exactly like any -other request — the only difference from the default-port examples above is -the port named in the URL and dialing the router's CONNECT listener (8081/8001 -here) instead of its plain HTTP one. +Then send a request to local port `9090`: + +```bash +curl http://localhost:9090/ +``` + +This reaches the same actor's second listener and resumes it like any other +request. **Current limitation**: only HTTP(S) traffic over the tunnel is supported. Raw TCP or other non-HTTP protocols on a non-default port aren't reachable diff --git a/demos/egress/README.md b/demos/egress/README.md index 78daf91713..923c0c2647 100644 --- a/demos/egress/README.md +++ b/demos/egress/README.md @@ -58,10 +58,12 @@ ActorTemplate, worker pool, test, and manual walkthrough are otherwise the same. ```bash # Envoy (default) -./hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' +./hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-system/sa/atenet-client # agentgateway -./hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=agentgateway --credential-provider='{"name":"k8s.io"}' +./hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=agentgateway --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-system/sa/atenet-client ``` | | Envoy | agentgateway | @@ -168,16 +170,13 @@ EOF # 4. Drive the Actor's egress through the ingress gateway. The gateway caches a # missing policy as deny for 10s (--egress-policy-cache-ttl), so a fetch tried # before step 3 keeps failing for up to that long after the policy appears. -kubectl -n ate-system port-forward service/atenet-router 8000:80 & +kubectl ate proxy actor egress-demo 8000:80 -a ate-demo-egress & curl -s -X POST http://localhost:8000/ \ - -H 'ate-target-actor: ate-demo-egress/egress-demo' \ -H 'Content-Type: application/json' \ -d "{\"url\":\"http://${TARGET_IP}:80/\"}" ``` -The `ate-target-actor` header selects the Actor receiving this ingress request. The -URL in the JSON body selects that Actor's egress destination and is unrelated -to Actor routing. +The URL in the JSON body selects the Actor's egress destination. ### What to observe @@ -186,7 +185,7 @@ With Envoy: ```bash # The egress gateway logs each tunneled CONNECT against the verified peer certificate: kubectl -n ate-system logs deploy/atenet-egress -c envoy | grep '\[egress\]' -# [egress] authority=:80 peer_san=spiffe://substrate-actor.local/atespace/ate-demo-egress/actor/egress-demo … code=200 … +# [egress] authority=:80 peer_san=spiffe://substrate-actor.local/worker-for-actor/ate-demo-egress/egress-demo … code=200 … # The co-located ext_proc sidecar logs the identity decision, including the UID it authorized on: kubectl -n ate-system logs deploy/atenet-egress -c ext-proc | grep -i 'egress tunnel opened\|egress denied' diff --git a/demos/egress/test-egress.sh b/demos/egress/test-egress.sh index 1d9c2fc48c..83a4cc3ae4 100755 --- a/demos/egress/test-egress.sh +++ b/demos/egress/test-egress.sh @@ -24,7 +24,8 @@ # CA, so the mTLS handshake itself is refused. # # Prerequisites: a substrate cluster with `--deploy-demo-egress` applied, plus -# kubectl and kubectl-ate on PATH. See demos/egress/README.md. +# kubectl and kubectl-ate on PATH. The install's --ingress-allowed-spiffe-ids +# must include spiffe://cluster.local/ns/ate-system/sa/atenet-client. See demos/egress/README.md. # # Usage: # demos/egress/test-egress.sh # run the tests @@ -118,10 +119,9 @@ wait_egress_log() { for _ in $(seq 1 10); do if egress_log_since "$1" | grep -qE log "POSITIVE — real Actor egress is tunneled through the gateway (expect 200)" ############################################################################## BEFORE=$(egress_log_count) -${K} -n ate-system port-forward service/atenet-router 18099:80 >/tmp/egress-pf.log 2>&1 & +${KATE} proxy actor "${ACTOR}" 18099:80 -a "${ATESPACE}" >/tmp/egress-pf.log 2>&1 & PF=$!; sleep 4 CODE=$(curl -s -o /tmp/egress-body.txt -w '%{http_code}' -X POST http://localhost:18099/ \ - -H "ate-target-actor: ${ATESPACE}/${ACTOR}" \ -H 'Content-Type: application/json' \ -d "{\"url\":\"http://${TARGET_IP}:${TARGET_PORT}/\"}" || true) kill "${PF}" >/dev/null 2>&1 || true @@ -137,7 +137,7 @@ fi if [[ "${DATAPLANE}" == "envoy" ]]; then # Envoy's access log identifies the peer by its certificate SAN, not by any # header the actor could have written. - ALLOWED_LOG_PATTERN="actor/${ACTOR}.*code=200" + ALLOWED_LOG_PATTERN="worker-for-actor/${ATESPACE}/${ACTOR}.*code=200" else # agentgateway's structured access log records the CONNECT authority. Actor # authentication is also covered by the successful request and negative test. diff --git a/demos/jupyter/README.md b/demos/jupyter/README.md index 36b2644004..8931321ab9 100644 --- a/demos/jupyter/README.md +++ b/demos/jupyter/README.md @@ -22,7 +22,8 @@ For local development, Agent Substrate provides helper scripts to get a Kind clu 2. **Install Agent Substrate core system to the Kind cluster:** ```bash - ./hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' + ./hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-system/sa/atenet-client ``` 3. **Deploy the Jupyter Notebook Demo:** @@ -53,30 +54,11 @@ kubectl ate create actor jupyter-notebook -a ate-demo-jupyter --template jupyter ### 2. Access Jupyter via the Proxy! -Substrate routes HTTP traffic using `ate-target-actor: /`. -A browser cannot add that header to an address-bar navigation or WebSocket -upgrade, so this demo puts a lightweight NGINX reverse proxy (`jupyter-proxy`) -between the browser and the Substrate router. Its configuration injects the -header for the Actor created above, preserves the browser's `Host` as application -metadata, and preserves Jupyter's WebSocket upgrade: - -```nginx -proxy_pass http://atenet-router.ate-system.svc.cluster.local; -proxy_set_header Host $http_host; -proxy_set_header ate-target-actor ate-demo-jupyter/jupyter-notebook; -proxy_set_header Upgrade $http_upgrade; -proxy_set_header Connection "upgrade"; -``` - -The complete configuration is in `jupyter.yaml.tmpl`. If you create the Actor -under another name or Atespace, update the `ate-target-actor` value before -deploying the proxy. Do not derive it from an untrusted browser-supplied header. - -1. **Port-forward the lightweight proxy to your local machine:** +1. **Start the local actor proxy:** Keep this running in a separate terminal: ```bash - kubectl port-forward --address 0.0.0.0 -n ate-demo-jupyter svc/jupyter-proxy 8888:80 + kubectl ate proxy actor jupyter-notebook 8888:80 -a ate-demo-jupyter ``` 2. **Open your browser!** diff --git a/demos/jupyter/jupyter.yaml.tmpl b/demos/jupyter/jupyter.yaml.tmpl index e68cdd9817..61d1194937 100644 --- a/demos/jupyter/jupyter.yaml.tmpl +++ b/demos/jupyter/jupyter.yaml.tmpl @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Worker pool and router-proxy plumbing for the jupyter demo. The -# ActorTemplate is not a CRD: it lives in jupyter-template.yaml.tmpl and is -# created through the ate API with `kubectl ate create actor-template`. - apiVersion: v1 kind: Namespace metadata: @@ -35,69 +31,3 @@ spec: workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: jupyter-proxy-config - namespace: ate-demo-jupyter -data: - nginx.conf: | - events {} - http { - server { - listen 80; - location / { - proxy_pass http://atenet-router.ate-system.svc.cluster.local; - proxy_set_header Host $http_host; - proxy_set_header ate-target-actor ate-demo-jupyter/jupyter-notebook; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - } - } ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: jupyter-proxy - namespace: ate-demo-jupyter -spec: - replicas: 1 - selector: - matchLabels: - app: jupyter-proxy - template: - metadata: - labels: - app: jupyter-proxy - spec: - containers: - - name: nginx - image: nginx:alpine - ports: - - containerPort: 80 - volumeMounts: - - name: config - mountPath: /etc/nginx/nginx.conf - subPath: nginx.conf - volumes: - - name: config - configMap: - name: jupyter-proxy-config ---- -apiVersion: v1 -kind: Service -metadata: - name: jupyter-proxy - namespace: ate-demo-jupyter -spec: - type: ClusterIP - selector: - app: jupyter-proxy - ports: - - port: 80 - targetPort: 80 diff --git a/demos/multi-template/README.md b/demos/multi-template/README.md index bbf8926608..0ac1e739e9 100644 --- a/demos/multi-template/README.md +++ b/demos/multi-template/README.md @@ -9,7 +9,8 @@ or namespace. ## Prerequisites -- A k8s cluster with Agent Substrate installed. +- A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) + or [GKE Quickstart](../../README.md#gke-quickstart-development). - `ko` installed for building images. - A GCS bucket for storing snapshots (configured via `BUCKET_NAME` env var). @@ -51,28 +52,26 @@ kubectl ate create actor c1 -a ate-demo-multi-template-counter --template counte kubectl ate create actor f1 -a ate-demo-multi-template-fspersist --template fspersist ``` -### 3. Port-forward the atenet router +### 3. Proxy to the actors -To interact with the router locally: +Start a local proxy for each actor: ```bash -kubectl port-forward -n ate-system svc/atenet-router 8000:80 +kubectl ate proxy actor c1 8000:80 -a ate-demo-multi-template-counter & +kubectl ate proxy actor f1 8001:80 -a ate-demo-multi-template-fspersist & ``` ## How to Use -When you send an HTTP request through the router, Substrate automatically detects the session, activates (resumes) the actor onto an available worker pod, and proxies the traffic. -The two headers form the complete routing key: the Actor name alone is not -enough, so each request below pairs it with the Atespace where that Actor was -created. +When you send an HTTP request through the proxy, `atenet-router` automatically activates (resumes) the actor onto an available worker pod and routes the traffic: ```bash # counter binary -curl -s -H "ate-target-actor: ate-demo-multi-template-counter/c1" http://localhost:8000 +curl -s http://localhost:8000 # -> hello from: | preserved memory count: 1 # fspersist binary -curl -s -H "ate-target-actor: ate-demo-multi-template-fspersist/f1" http://localhost:8000 +curl -s http://localhost:8001 # -> pod: # --- history --- # pod= | count=0 | time= @@ -90,7 +89,7 @@ preserves that state across the snapshot/restore cycle: ```bash kubectl ate suspend actor f1 -a ate-demo-multi-template-fspersist -curl -s -H "ate-target-actor: ate-demo-multi-template-fspersist/f1" http://localhost:8000 # history persists; count keeps climbing +curl -s http://localhost:8001 # history persists; count keeps climbing ``` ## How to Uninstall diff --git a/demos/parking/README.md b/demos/parking/README.md index 22a95c12ef..fc0aa5c6b4 100644 --- a/demos/parking/README.md +++ b/demos/parking/README.md @@ -13,7 +13,8 @@ See [docs/request-parking.md](../../docs/request-parking.md) for the design. ## Prerequisites -- A k8s cluster with Agent Substrate installed. +- A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) + or [GKE Quickstart](../../README.md#gke-quickstart-development). - `ko` installed for building images. - A GCS bucket for storing snapshots (configured via `BUCKET_NAME` env var). @@ -53,37 +54,36 @@ for id in p1 p2 p3 p4; do done ``` -### 3. Port-forward the atenet router +### 3. Proxy to the actors ```bash -kubectl port-forward -n ate-system svc/atenet-router 8000:80 +kubectl ate proxy actor p1 8001:80 -a ate-demo-parking & +kubectl ate proxy actor p2 8002:80 -a ate-demo-parking & +kubectl ate proxy actor p3 8003:80 -a ate-demo-parking & ``` ## How to Use Parking is **on by default** (`--parked-request-budget=5s`, `--parked-request-max=1024`), so the cluster you just deployed already parks. -Every curl request below selects its Actor with `ate-target-actor`; changing -the URL or `Host` does not select another Actor. ### A. Watch a 503 become a served request Fill both workers by requesting two actors, leaving them `RUNNING`: ```bash -curl -s -H "ate-target-actor: ate-demo-parking/p1" http://localhost:8000 -curl -s -H "ate-target-actor: ate-demo-parking/p2" http://localhost:8000 +curl -s http://localhost:8001 +curl -s http://localhost:8002 kubectl ate get workers # both workers are now bound to p1 and p2 -kubectl ate get actors # p1,p2 RUNNING; p3,p4 SUSPENDED +kubectl ate get actors -a ate-demo-parking # p1,p2 RUNNING; p3,p4 SUSPENDED ``` Now request **p3** with timing. The pool is full, so this request **parks** — the `curl` hangs while the router retries the resume: ```bash -curl -s -w '\n-> HTTP %{http_code} in %{time_total}s\n' \ - -H "ate-target-actor: ate-demo-parking/p3" http://localhost:8000 +curl -s -w '\n-> HTTP %{http_code} in %{time_total}s\n' http://localhost:8003 ``` While that is hanging, in a **second terminal** free a worker by suspending p1 diff --git a/demos/parking/load.sh b/demos/parking/load.sh index 3eae8a13a4..bf8d6d3f95 100755 --- a/demos/parking/load.sh +++ b/demos/parking/load.sh @@ -29,43 +29,41 @@ # See README.md for how to flip the router flag. # # Usage: -# ./load.sh [-d duration_secs] [-r router_url] [-a atespace] [actor_id ...] +# ./load.sh [-d duration_secs] [-a atespace] [actor_id ...] # # Examples: -# ./load.sh # 30s, actors p1 p2 p3 p4, http://localhost:8000 +# ./load.sh # 30s, actors p1 p2 p3 p4 # ./load.sh -d 60 p1 p2 p3 p4 p5 p6 # # Prerequisites: # * `kubectl ate` plugin installed (go install ./cmd/kubectl-ate) -# * router port-forwarded: kubectl port-forward -n ate-system svc/atenet-router 8000:80 +# * an install whose --ingress-allowed-spiffe-ids includes +# spiffe://cluster.local/ns/ate-system/sa/atenet-client set -uo pipefail DURATION=30 -ROUTER="http://localhost:8000" # The template name, resolved in the actors' atespace (--template). TEMPLATE="parking" ATESPACE="ate-demo-parking" +BASE_PORT=18000 usage() { cat <<'EOF' load.sh -- oversubscription load generator for the request-parking demo. -Usage: ./load.sh [-d duration_secs] [-r router_url] [-a atespace] [actor_id ...] +Usage: ./load.sh [-d duration_secs] [-a atespace] [actor_id ...] -d load duration in seconds (default 30) - -r router base URL (default http://localhost:8000) -a atespace for the actors (default ate-demo-parking) args actor IDs (default: p1 p2 p3 p4) -Prereqs: `kubectl ate` installed and the router port-forwarded - (kubectl port-forward -n ate-system svc/atenet-router 8000:80). +Prereqs: `kubectl ate` installed. EOF } -while getopts ":d:r:a:h" opt; do +while getopts ":d:a:h" opt; do case "${opt}" in d) DURATION="${OPTARG}" ;; - r) ROUTER="${OPTARG}" ;; a) ATESPACE="${OPTARG}" ;; h) usage; exit 0 ;; *) echo "unknown option -${OPTARG}; use -h for help" >&2; exit 2 ;; @@ -80,13 +78,20 @@ fi TMP="$(mktemp -d)" pids=() -cleanup() { rm -rf "${TMP}"; } -abort() { [[ ${#pids[@]} -gt 0 ]] && kill "${pids[@]}" 2>/dev/null; cleanup; exit 130; } +proxy_pids=() +cleanup() { + [[ ${#proxy_pids[@]} -gt 0 ]] && kill "${proxy_pids[@]}" 2>/dev/null + rm -rf "${TMP}" +} +abort() { + [[ ${#pids[@]} -gt 0 ]] && kill "${pids[@]}" 2>/dev/null + cleanup + exit 130 +} trap cleanup EXIT trap abort INT TERM echo "==> request-parking load test" -echo " router: ${ROUTER}" echo " duration: ${DURATION}s" echo " actors: ${ACTORS[*]} (${#ACTORS[@]}) vs a 2-worker pool -> oversubscribed" echo @@ -99,26 +104,36 @@ for a in "${ACTORS[@]}"; do kubectl ate create actor "${a}" --atespace "${ATESPACE}" --template "${TEMPLATE}" >/dev/null 2>&1 || true done +echo "==> starting local proxies for actors ..." +for i in "${!ACTORS[@]}"; do + a="${ACTORS[$i]}" + port=$((BASE_PORT + i + 1)) + kubectl ate proxy actor "${a}" "${port}:80" --atespace "${ATESPACE}" >"${TMP}/${a}.proxy.log" 2>&1 & + proxy_pids+=($!) +done +sleep 3 + # One worker per actor: hammer it with request->suspend until the deadline. worker() { - local actor="$1" log="${TMP}/$1.log" + local actor="$1" port="$2" log="${TMP}/$1.log" local deadline=$(( $(date +%s) + DURATION )) while [[ $(date +%s) -lt ${deadline} ]]; do # %{http_code} lets us tally outcomes; %{time_total} reveals parking waits. curl -s -o /dev/null -w '%{http_code} %{time_total}\n' \ - -H "ate-target-actor: ${ATESPACE}/${actor}" \ - "${ROUTER}" >>"${log}" 2>/dev/null + "http://127.0.0.1:${port}" >>"${log}" 2>/dev/null # Free the worker so a parked competitor can proceed (simulate going idle). kubectl ate suspend actor "${actor}" --atespace "${ATESPACE}" >/dev/null 2>&1 || true done } echo "==> generating load for ${DURATION}s ..." -for a in "${ACTORS[@]}"; do - worker "${a}" & +for i in "${!ACTORS[@]}"; do + a="${ACTORS[$i]}" + port=$((BASE_PORT + i + 1)) + worker "${a}" "${port}" & pids+=($!) done -wait +wait "${pids[@]}" # ---- aggregate ------------------------------------------------------------- echo @@ -126,7 +141,7 @@ echo "==> results" cat "${TMP}"/*.log 2>/dev/null >"${TMP}/all" || true total=$(wc -l <"${TMP}/all" | tr -d ' ') if [[ "${total}" -eq 0 ]]; then - echo " no responses recorded -- is the router port-forwarded at ${ROUTER}?" + echo " no responses recorded -- check proxy logs" exit 1 fi diff --git a/demos/sandbox/README.md b/demos/sandbox/README.md index 4be9024390..843bef752e 100644 --- a/demos/sandbox/README.md +++ b/demos/sandbox/README.md @@ -14,7 +14,8 @@ It allows you to run arbitrary commands in an sandboxed, isolated container (run ## Prerequisites -- A k8s cluster with Agent Substrate installed. +- A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) + or [GKE Quickstart](../../README.md#gke-quickstart-development). - `ko` installed for building images. - A GCS bucket for storing snapshots (configured in `demos/sandbox/sandbox-template.yaml.tmpl`). - `kubectl-ate` CLI installed (can be installed via `go install ./cmd/kubectl-ate`). @@ -56,16 +57,12 @@ go install ./cmd/kubectl-ate kubectl ate create actor my-sandbox-1 -a ate-demo-sandbox --template sandbox-template ``` -### 3. Port-Forward Services +### 3. Proxy to the Actor -If running clients locally, port-forward the API and router in separate terminals: +Start a local proxy to the actor in a separate terminal: ```bash -# Terminal 1: API Server -kubectl port-forward -n ate-system svc/api 8080:443 - -# Terminal 2: Router -kubectl port-forward -n ate-system svc/atenet-router 8000:80 +kubectl ate proxy actor my-sandbox-1 -a ate-demo-sandbox ``` ## How to Use the Client @@ -75,19 +72,9 @@ Build and run the client REPL: ```bash go build -o bin/sandbox-client ./demos/sandbox/client -./bin/sandbox-client --ateapi=localhost:8080 --atenet=localhost:8000 --atespace=ate-demo-sandbox --name=my-sandbox-1 +./bin/sandbox-client --atenet=localhost:8000 --atespace=ate-demo-sandbox --name=my-sandbox-1 ``` -The client sends each `/process` request to the router and automatically sets -the routing header from `--name` and `--atespace`: - -```go -req.Header.Set(atenet.TargetActorHeader, actorRef.String()) -``` - -Any replacement HTTP client must send an equivalent `ate-target-actor` header; -the URL and `Host` header do not select the Actor. - Once in the `sandbox>` prompt, you can run commands: ```bash diff --git a/demos/sandbox/client/main.go b/demos/sandbox/client/main.go index 186b35ff90..14d1347bae 100644 --- a/demos/sandbox/client/main.go +++ b/demos/sandbox/client/main.go @@ -17,7 +17,6 @@ import ( "bufio" "bytes" "context" - "crypto/tls" "encoding/json" "fmt" "io" @@ -30,10 +29,9 @@ import ( "github.com/spf13/pflag" "go.ate.dev/substrate/api/ateapi/v1" + "go.ate.dev/substrate/internal/ateclient" "go.ate.dev/substrate/internal/atenet" "go.ate.dev/substrate/internal/resources" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) type ProcessRequest struct { @@ -49,22 +47,13 @@ type ProcessResponse struct { Error string `json:"error,omitempty"` } -func dialAteAPI(endpoint string) (ateapi.ControlClient, *grpc.ClientConn, error) { - creds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) - - conn, err := grpc.NewClient(endpoint, grpc.WithTransportCredentials(creds)) - if err != nil { - return nil, nil, err - } - - return ateapi.NewControlClient(conn), conn, nil -} - func main() { actorName := pflag.String("name", "", "Name of the sandbox actor (required)") atespace := pflag.String("atespace", "", "Atespace the actor lives in (required)") - ateapiAddr := pflag.String("ateapi", "localhost:8080", "Address of the ateapi gRPC server") - atenetAddr := pflag.String("atenet", "localhost:8000", "Address of the atenet HTTP router") + ateapiAddr := pflag.String("ateapi", "", "Address of the ateapi gRPC server (defaults to port-forwarding svc/api)") + atenetAddr := pflag.String("atenet", "localhost:8000", "Address of the local actor proxy") + kubeconfig := pflag.String("kubeconfig", "", "Path to the kubeconfig file") + k8sContext := pflag.String("context", "", "The name of the kubeconfig context to use") pflag.Parse() if *actorName == "" { @@ -88,12 +77,12 @@ func main() { }() // 1. Connect to ateapi and Resume Actor - log.Printf("Connecting to ateapi at %s...", *ateapiAddr) - cli, conn, err := dialAteAPI(*ateapiAddr) + log.Println("Connecting to ateapi...") + cli, err := ateclient.NewClient(ctx, *kubeconfig, *k8sContext, *ateapiAddr, "", false) if err != nil { log.Fatalf("Failed to dial ateapi: %v", err) } - defer conn.Close() + defer cli.Close() log.Printf("Resuming actor %s...", actorRef.Name) _, err = cli.ResumeActor(ctx, &ateapi.ResumeActorRequest{Actor: actorRef.ToObjectRef()}) diff --git a/docs/api-guide.md b/docs/api-guide.md index ffb19a45ff..fdf3292b57 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -185,20 +185,36 @@ the lowercase name `ate-target-actor`. For an HTTP `CONNECT` tunnel to a non-default Actor port, put the routing header on the outer `CONNECT` request and keep the target port in its -authority. With curl, use `--proxy-header` instead of `-H`: +authority. With curl and the projected volumes from the router's +[common setups](../cmd/atenet/internal/router/README.md#common-setups), use +`--proxy-header` instead of `-H`: ```bash -curl --proxytunnel --proxy http://localhost:8001 \ +curl --proxytunnel \ + --proxy https://atenet-router.ate-system.svc:8444 \ + --proxy-cacert /run/router-client/router-ca.pem \ + --proxy-cert /run/router-client/credential-bundle.pem \ + --proxy-key /run/router-client/credential-bundle.pem \ --proxy-header "ate-target-actor: my-atespace/my-actor" \ http://actor-upstream:9090/ ``` -Browser navigation cannot add custom request headers. Browser-based and other -fixed clients must therefore connect through a user-controlled reverse proxy or -policy enforcement point that overwrites the routing header before forwarding -to `atenet-router`. The [Jupyter demo](../demos/jupyter/README.md) shows this -pattern with NGINX. The proxy must derive the value from trusted configuration -or authenticated request context rather than forwarding values supplied by an +From outside the cluster, `kubectl ate proxy actor` opens this mTLS `CONNECT` +tunnel automatically: + +```bash +kubectl ate proxy actor my-actor 9090:9090 -a my-atespace +curl http://localhost:9090/ +``` + +Browser navigation cannot add custom request headers or present the router's +client certificate. For local development and demos (such as the +[Jupyter demo](../demos/jupyter/README.md)), `kubectl ate proxy actor` serves a +local port that forwards browser traffic and WebSockets through `atenet-router`. +In production, browser-based and other fixed clients connect through a +user-controlled reverse proxy or policy enforcement point that authenticates to +`atenet-router` and sets `ate-target-actor` from trusted configuration or +authenticated request context rather than forwarding values supplied by an untrusted caller. ### SystemInfo Volumes diff --git a/docs/operator-install.md b/docs/operator-install.md index b32ed3d0a9..15b649a8a3 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -370,6 +370,19 @@ any other bucket a template names, as setup-gcp does on the snapshot bucket. Wit ate-api-server ignores `snapshotConfig.storageLocation`. See [Snapshot Storage Layout](api-guide.md#snapshot-storage-layout). +## Ingress to actors + +> [!IMPORTANT] +> atenet-router lets a client reach actors only if the client's SPIFFE ID is +> in `ingressAllowedSPIFFEIDs` (`--ingress-allowed-spiffe-ids`). The setting +> has no default, so an install that leaves it unset accepts no ingress +> traffic. + +List every client that calls atenet-router. If all traffic reaches actors +through one frontend or gateway, list that gateway's identity. If several +services call the router, list each of them. Any listed client may reach any +actor. + ## Secrets Credentials should come from the environment, not from a document that gets diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 995cb028f6..85baf6c4ea 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -30,7 +30,8 @@ # requires a credential provider selection, which reaches it only through the # environment here: export ATE_CREDENTIAL_PROVIDER (e.g. '{"name":"k8s.io"}'; # see hack/install-ate.sh --help) before running this, the kind wrapper -# included. +# included. The printed `kubectl ate proxy actor` step also needs +# ATE_INGRESS_ALLOWED_SPIFFE_IDS to include spiffe://cluster.local/ns/ate-system/sa/atenet-client. # # Env (most come from .ate-dev-env.sh): # KO_DOCKER_REPO (required) image registry, e.g. gcr.io/PROJECT/ate-images for @@ -121,11 +122,9 @@ cat </dev/null | wc -l | tr -d ' ') echo "== drive actor egress: GET ${TARGET_URL} via the actor ==" -${K} -n ate-system port-forward service/atenet-router 18000:80 >/tmp/pf.log 2>&1 & +kubectl-ate --context "${CTX}" proxy actor "${ACTOR}" 18000:80 --atespace "${ATESPACE}" >/tmp/pf.log 2>&1 & PF=$!; trap 'kill ${PF} 2>/dev/null || true' EXIT sleep 3 RESP=$(curl -s -o /dev/null -w "%{http_code}" -X POST http://localhost:18000/ \ - -H "ate-target-actor: ${ATESPACE}/${ACTOR}" \ -H 'Content-Type: application/json' \ -d "{\"url\":\"${TARGET_URL}\"}") || true echo "actor round-trip HTTP ${RESP} (200 = the actor fetched ${TARGET_URL} through egress)" diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index f267e2dedc..4507b1a3f7 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -184,7 +184,7 @@ func dialDirect(ctx context.Context, kubeconfigPath, k8sContext, endpoint, token // Verify the server before attaching the bearer token below: the token // must never be sent over an unauthenticated channel. - tlsCfg, err := serverTLSConfig(ctx, clientset) + tlsCfg, err := serverTLSConfig(ctx, clientset, apiServerName()) if err != nil { return nil, err } @@ -246,7 +246,7 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext, tokenFile } localEndpoint := fmt.Sprintf("127.0.0.1:%d", localPort) - tlsCfg, err := serverTLSConfig(ctx, clientset) + tlsCfg, err := serverTLSConfig(ctx, clientset, apiServerName()) if err != nil { stopForward() return nil, err @@ -280,7 +280,9 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext, tokenFile }, nil } -func serverTLSConfig(ctx context.Context, clientset kubernetes.Interface) (*tls.Config, error) { +// serverTLSConfig verifies a Service's servicedns serving certificate for +// serverName. +func serverTLSConfig(ctx context.Context, clientset kubernetes.Interface, serverName string) (*tls.Config, error) { bundles, err := clustertrustbundle.NewClient(clientset, nil) if err != nil { return nil, fmt.Errorf("failed to discover ClusterTrustBundle API: %w", err) @@ -310,7 +312,7 @@ func serverTLSConfig(ctx context.Context, clientset kubernetes.Interface) (*tls. return &tls.Config{ MinVersion: tls.VersionTLS13, RootCAs: pool, - ServerName: apiServerName(), + ServerName: serverName, }, nil } diff --git a/internal/ateclient/builder_test.go b/internal/ateclient/builder_test.go index e6d4cff25f..e9caea3d6c 100644 --- a/internal/ateclient/builder_test.go +++ b/internal/ateclient/builder_test.go @@ -155,7 +155,7 @@ func TestServerTLSConfig(t *testing.T) { } clientset := fake.NewSimpleClientset(native...) clientset.Resources = []*metav1.APIResourceList{{GroupVersion: "certificates.k8s.io/" + version, APIResources: []metav1.APIResource{{Name: "clustertrustbundles"}}}} - cfg, err := serverTLSConfig(context.Background(), clientset) + cfg, err := serverTLSConfig(context.Background(), clientset, apiServerName()) if err != nil { t.Fatalf("serverTLSConfig: %v", err) } @@ -204,7 +204,7 @@ func TestServerTLSConfigErrors(t *testing.T) { t.Run(tc.name, func(t *testing.T) { clientset := fake.NewSimpleClientset(tc.objects...) clientset.Resources = []*metav1.APIResourceList{{GroupVersion: "certificates.k8s.io/v1", APIResources: []metav1.APIResource{{Name: "clustertrustbundles"}}}} - if _, err := serverTLSConfig(context.Background(), clientset); err == nil { + if _, err := serverTLSConfig(context.Background(), clientset, apiServerName()); err == nil { t.Error("serverTLSConfig: want error, got nil") } }) diff --git a/internal/ateclient/router.go b/internal/ateclient/router.go new file mode 100644 index 0000000000..1ffaaaee21 --- /dev/null +++ b/internal/ateclient/router.go @@ -0,0 +1,266 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateclient + +import ( + "bufio" + "context" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" + "time" + + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/installdefaults" + "go.ate.dev/substrate/internal/podidentity" + "go.ate.dev/substrate/internal/portforward" + "go.ate.dev/substrate/internal/resources" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +const ( + // routerConnectTLSPort is atenet-router's CONNECT-TLS Service port. + routerConnectTLSPort = 8444 + // routerCertTimeout bounds the initial wait for the podidentity signer. + routerCertTimeout = 2 * time.Minute + // routerCertRefreshTimeout bounds a refresh attempt made during a + // handshake while a cached certificate is still valid. + routerCertRefreshTimeout = 15 * time.Second + // routerCertRequester names this client on its PodCertificateRequests. + routerCertRequester = "kubectl-ate" + // routerClientServiceAccount is the ServiceAccount the client requests its + // router client certificate for. The install must allow its SPIFFE ID. + routerClientServiceAccount = "atenet-client" +) + +func routerServerName() string { + return fmt.Sprintf("%s.%s.svc", installdefaults.RouterServiceName, systemNamespace()) +} + +// RouterDialer opens CONNECT tunnels to actors through atenet-router. +type RouterDialer struct { + config *rest.Config + clientset kubernetes.Interface + tlsConfig *tls.Config +} + +// NewRouterDialer returns a dialer that reaches atenet-router's CONNECT-TLS +// listener through a port-forward. It verifies the router's servicedns serving +// certificate and presents a podidentity certificate for the ingress client +// ServiceAccount, which the router allows by default. +// +// The certificate is requested before NewRouterDialer returns, so a caller +// without RBAC to create PodCertificateRequests fails here. It is requested +// again once half its lifetime has passed. +// +// TODO: Authenticate with a JWT once atenet-router supports JWT-based +// authentication instead of minting a client certificate via PodCertificateRequest. +func NewRouterDialer(ctx context.Context, kubeconfigPath, k8sContext string) (*RouterDialer, error) { + config, err := LoadKubeConfig(kubeconfigPath, k8sContext) + if err != nil { + return nil, fmt.Errorf("failed to load kubeconfig: %w", err) + } + // Older clusters use the deprecated beta ClusterTrustBundle and + // PodCertificateRequest APIs. Suppress their warnings when falling back + // from v1. + config.WarningHandlerWithContext = &rest.NoWarnings{} + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create k8s client: %w", err) + } + + tlsCfg, err := serverTLSConfig(ctx, clientset, routerServerName()) + if err != nil { + return nil, err + } + certs := &routerClientCerts{clientset: clientset} + if _, err := certs.get(ctx); err != nil { + return nil, err + } + tlsCfg.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { + return certs.get(info.Context()) + } + + return &RouterDialer{ + config: config, + clientset: clientset, + tlsConfig: tlsCfg, + }, nil +} + +// DialActor opens a CONNECT tunnel through atenet-router to port on actorRef. +// Closing the returned connection also stops its port-forward. +func (d *RouterDialer) DialActor(ctx context.Context, actorRef resources.ActorRef, port int) (net.Conn, error) { + // client-go tears down a whole port-forward when any one of its + // streams errors, so each tunnel gets its own. That also picks + // a ready router pod again after one goes away. + localPort, stop, err := portforward.ServicePortForward(ctx, d.config, d.clientset, systemNamespace(), installdefaults.RouterServiceName, routerConnectTLSPort) + if err != nil { + return nil, err + } + return ConnectActorWithHeaders(ctx, fmt.Sprintf("127.0.0.1:%d", localPort), d.tlsConfig, actorRef, port, http.Header{ + atenet.TargetActorHeader: []string{actorRef.String()}, + }, stop) +} + +// ConnectActorWithHeaders dials addr over TLS, sends an HTTP CONNECT request +// for actorRef:port with header, and returns the tunnel connection. Closing +// the returned connection calls stop at most once if non-nil. +func ConnectActorWithHeaders(ctx context.Context, addr string, tlsCfg *tls.Config, actorRef resources.ActorRef, port int, header http.Header, stop func()) (net.Conn, error) { + var dialer net.Dialer + rawConn, err := dialer.DialContext(ctx, "tcp", addr) + if err != nil { + if stop != nil { + stop() + } + return nil, fmt.Errorf("connecting to router's CONNECT listener: %w", err) + } + conn, err := connectActor(ctx, rawConn, tlsCfg, actorRef, port, header) + if err != nil { + if stop != nil { + stop() + } + return nil, err + } + conn.stop = stop + return conn, nil +} + +func connectActor(ctx context.Context, rawConn net.Conn, tlsCfg *tls.Config, actorRef resources.ActorRef, port int, header http.Header) (*forwardedConn, error) { + tlsConn := tls.Client(rawConn, tlsCfg.Clone()) + if err := tlsConn.HandshakeContext(ctx); err != nil { + _ = rawConn.Close() + return nil, fmt.Errorf("TLS handshake with router's CONNECT listener: %w", err) + } + + stopOnCancel := context.AfterFunc(ctx, func() { _ = tlsConn.Close() }) + defer stopOnCancel() + + destination := net.JoinHostPort(actorRef.Name, strconv.Itoa(port)) + req := &http.Request{ + Method: http.MethodConnect, + URL: &url.URL{Host: destination}, + Host: destination, + Header: header, + } + if err := req.Write(tlsConn); err != nil { + _ = tlsConn.Close() + return nil, fmt.Errorf("writing CONNECT request: %w", err) + } + + reader := bufio.NewReader(tlsConn) + resp, err := http.ReadResponse(reader, req) + if err != nil { + _ = tlsConn.Close() + return nil, fmt.Errorf("reading CONNECT response: %w", err) + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) + _ = resp.Body.Close() + _ = tlsConn.Close() + message := strings.TrimSpace(string(body)) + if message == "" { + message = resp.Status + } + return nil, fmt.Errorf("router rejected CONNECT to %s with %s: %s", destination, resp.Status, message) + } + if !stopOnCancel() { + _ = tlsConn.Close() + return nil, ctx.Err() + } + + return &forwardedConn{Conn: tlsConn, reader: reader}, nil +} + +// routerClientCerts holds the current router client certificate. +type routerClientCerts struct { + clientset kubernetes.Interface + + mu sync.Mutex + cert *tls.Certificate + refreshAt time.Time + warnedFallback bool +} + +func (c *routerClientCerts) get(ctx context.Context) (*tls.Certificate, error) { + c.mu.Lock() + defer c.mu.Unlock() + now := time.Now() + if c.cert != nil && now.Before(c.refreshAt) { + return c.cert, nil + } + timeout := routerCertTimeout + if c.cert != nil && now.Before(c.cert.Leaf.NotAfter) { + timeout = routerCertRefreshTimeout + } + namespace := systemNamespace() + cert, err := podidentity.RequestCertificate(ctx, c.clientset, routerCertRequester, namespace, routerClientServiceAccount, timeout) + if err != nil { + now = time.Now() + if c.cert != nil && now.Before(c.cert.Leaf.NotAfter) { + if !c.warnedFallback { + c.warnedFallback = true + fmt.Fprintf(os.Stderr, "Warning: failed to refresh router client certificate (%v); using cached certificate until %s\n", err, c.cert.Leaf.NotAfter.Format(time.RFC3339)) + } + backoff := now.Add(time.Minute) + if c.cert.Leaf.NotAfter.Before(backoff) { + backoff = c.cert.Leaf.NotAfter + } + c.refreshAt = backoff + return c.cert, nil + } + return nil, fmt.Errorf("requesting a router client certificate for %s/%s: %w", namespace, routerClientServiceAccount, err) + } + c.cert = &cert + c.refreshAt = cert.Leaf.NotBefore.Add(cert.Leaf.NotAfter.Sub(cert.Leaf.NotBefore) / 2) + c.warnedFallback = false + return c.cert, nil +} + +// forwardedConn recovers bytes http.ReadResponse buffered past the header +// boundary and stops its port-forward when closed. +type forwardedConn struct { + net.Conn + reader *bufio.Reader + stop func() + stopOnce sync.Once +} + +func (c *forwardedConn) Read(p []byte) (int, error) { + return c.reader.Read(p) +} + +func (c *forwardedConn) CloseWrite() error { + if cw, ok := c.Conn.(interface{ CloseWrite() error }); ok { + return cw.CloseWrite() + } + return nil +} + +func (c *forwardedConn) Close() error { + err := c.Conn.Close() + if c.stop != nil { + c.stopOnce.Do(c.stop) + } + return err +} diff --git a/internal/ateclient/router_test.go b/internal/ateclient/router_test.go new file mode 100644 index 0000000000..feed0ffb01 --- /dev/null +++ b/internal/ateclient/router_test.go @@ -0,0 +1,294 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateclient + +import ( + "bufio" + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "io" + "math/big" + "net" + "net/http" + "slices" + "strings" + "testing" + "time" + + "go.ate.dev/substrate/internal/atenet" + "go.ate.dev/substrate/internal/resources" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +func testTLSConfigs(t *testing.T) (serverCfg, clientCfg *tls.Config) { + t.Helper() + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: routerServerName()}, + DNSNames: []string{routerServerName()}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, pub, priv) + if err != nil { + t.Fatal(err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + roots.AddCert(cert) + + serverCfg = &tls.Config{ + MinVersion: tls.VersionTLS13, + Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: priv}}, + } + clientCfg = &tls.Config{ + MinVersion: tls.VersionTLS13, + RootCAs: roots, + ServerName: routerServerName(), + } + return serverCfg, clientCfg +} + +func testTCPPair(t *testing.T) (clientConn, serverConn net.Conn) { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer lis.Close() + + accepted := make(chan net.Conn, 1) + go func() { + conn, err := lis.Accept() + if err != nil { + close(accepted) + return + } + accepted <- conn + }() + + c, err := net.Dial("tcp", lis.Addr().String()) + if err != nil { + t.Fatal(err) + } + s := <-accepted + t.Cleanup(func() { + _ = c.Close() + if s != nil { + _ = s.Close() + } + }) + return c, s +} + +func TestConnectActor(t *testing.T) { + serverTLS, clientTLS := testTLSConfigs(t) + clientRaw, serverRaw := testTCPPair(t) + + var gotReq *http.Request + serverDone := make(chan error, 1) + go func() { + defer close(serverDone) + tlsConn := tls.Server(serverRaw, serverTLS) + defer tlsConn.Close() + reader := bufio.NewReader(tlsConn) + req, err := http.ReadRequest(reader) + if err != nil { + serverDone <- err + return + } + gotReq = req + if _, err := io.WriteString(tlsConn, "HTTP/1.1 200 OK\r\n\r\nhello from actor"); err != nil { + serverDone <- err + return + } + gotClient, err := io.ReadAll(reader) + if err != nil { + serverDone <- err + return + } + if string(gotClient) != "ping" { + serverDone <- io.ErrUnexpectedEOF + } + }() + + actorRef := resources.ActorRef{Atespace: "team", Name: "my-counter"} + conn, err := connectActor(context.Background(), clientRaw, clientTLS, actorRef, 9090, http.Header{ + atenet.TargetActorHeader: []string{actorRef.String()}, + }) + if err != nil { + t.Fatalf("connectActor: %v", err) + } + stopped := false + conn.stop = func() { stopped = true } + + if _, err := io.WriteString(conn, "ping"); err != nil { + t.Fatalf("Write: %v", err) + } + if err := conn.CloseWrite(); err != nil { + t.Fatalf("CloseWrite: %v", err) + } + got, err := io.ReadAll(conn) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if err := conn.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if err := <-serverDone; err != nil { + t.Fatalf("server: %v", err) + } + + if gotReq.Method != http.MethodConnect { + t.Errorf("Method = %q, want CONNECT", gotReq.Method) + } + if gotReq.Host != "my-counter:9090" { + t.Errorf("Host = %q, want my-counter:9090", gotReq.Host) + } + if values, want := gotReq.Header.Values(atenet.TargetActorHeader), []string{"team/my-counter"}; !slices.Equal(values, want) { + t.Errorf("%s = %q, want %q", atenet.TargetActorHeader, values, want) + } + if string(got) != "hello from actor" { + t.Errorf("read = %q, want %q", got, "hello from actor") + } + if !stopped { + t.Error("Close did not stop the port-forward") + } +} + +func TestConnectActorRejected(t *testing.T) { + serverTLS, clientTLS := testTLSConfigs(t) + clientRaw, serverRaw := testTCPPair(t) + + go func() { + tlsConn := tls.Server(serverRaw, serverTLS) + defer tlsConn.Close() + _, _ = http.ReadRequest(bufio.NewReader(tlsConn)) + _, _ = io.WriteString(tlsConn, "HTTP/1.1 403 Forbidden\r\nContent-Length: 13\r\n\r\naccess denied") + }() + + actorRef := resources.ActorRef{Atespace: "team", Name: "my-counter"} + _, err := connectActor(context.Background(), clientRaw, clientTLS, actorRef, 80, http.Header{ + atenet.TargetActorHeader: []string{actorRef.String()}, + }) + if err == nil || !strings.Contains(err.Error(), "403 Forbidden: access denied") { + t.Fatalf("connectActor error = %v, want 403 Forbidden: access denied", err) + } +} + +func TestConnectActorContextCanceled(t *testing.T) { + serverTLS, clientTLS := testTLSConfigs(t) + clientRaw, serverRaw := testTCPPair(t) + + ctx, cancel := context.WithCancel(context.Background()) + unblockServer := make(chan struct{}) + defer close(unblockServer) + go func() { + tlsConn := tls.Server(serverRaw, serverTLS) + defer tlsConn.Close() + _, _ = http.ReadRequest(bufio.NewReader(tlsConn)) + cancel() + <-unblockServer + }() + + actorRef := resources.ActorRef{Atespace: "team", Name: "my-counter"} + _, err := connectActor(ctx, clientRaw, clientTLS, actorRef, 80, http.Header{ + atenet.TargetActorHeader: []string{actorRef.String()}, + }) + if err == nil { + t.Fatal("connectActor succeeded after context cancellation, want error") + } +} + +func TestRouterClientCertsFallback(t *testing.T) { + clientset := fake.NewClientset() + clientset.Resources = []*metav1.APIResourceList{{ + GroupVersion: "certificates.k8s.io/v1", + APIResources: []metav1.APIResource{{Name: "podcertificaterequests", Kind: "PodCertificateRequest", Namespaced: true}}, + }} + createCalls := 0 + clientset.PrependReactor("create", "podcertificaterequests", func(k8stesting.Action) (bool, runtime.Object, error) { + createCalls++ + return true, nil, io.ErrUnexpectedEOF + }) + + cached := &tls.Certificate{ + Leaf: &x509.Certificate{ + NotBefore: time.Now().Add(-40 * time.Minute), + NotAfter: time.Now().Add(20 * time.Minute), + }, + } + certs := &routerClientCerts{ + clientset: clientset, + cert: cached, + refreshAt: time.Now().Add(-10 * time.Minute), + } + + got, err := certs.get(context.Background()) + if err != nil { + t.Fatalf("get with unexpired cached cert: %v", err) + } + if got != cached { + t.Fatal("get did not fall back to cached certificate") + } + if createCalls != 1 { + t.Fatalf("createCalls = %d, want 1", createCalls) + } + + // A second call during the backoff window must reuse the cached cert + // without hitting the API server again. + if _, err := certs.get(context.Background()); err != nil { + t.Fatalf("second get during backoff: %v", err) + } + if createCalls != 1 { + t.Fatalf("createCalls after backoff call = %d, want 1", createCalls) + } + + // If NotAfter is sooner than the 1-minute backoff, refreshAt is capped + // at NotAfter so an expired cert is never handed out. + shortNotAfter := time.Now().Add(5 * time.Second) + certs.cert.Leaf.NotAfter = shortNotAfter + certs.refreshAt = time.Now().Add(-1 * time.Minute) + if _, err := certs.get(context.Background()); err != nil { + t.Fatalf("get with short NotAfter: %v", err) + } + if !certs.refreshAt.Equal(shortNotAfter) { + t.Fatalf("refreshAt = %v, want capped at NotAfter %v", certs.refreshAt, shortNotAfter) + } + + certs.refreshAt = time.Now().Add(-1 * time.Minute) + certs.cert.Leaf.NotAfter = time.Now().Add(-1 * time.Minute) + if _, err := certs.get(context.Background()); err == nil { + t.Fatal("get succeeded with expired cached cert, want error") + } +} diff --git a/internal/e2e/ingressauth.go b/internal/e2e/ingressauth.go index a83402bb71..a245b05d92 100644 --- a/internal/e2e/ingressauth.go +++ b/internal/e2e/ingressauth.go @@ -16,30 +16,20 @@ package e2e import ( "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" "crypto/tls" "crypto/x509" - "encoding/pem" "fmt" - "os" - "strings" "sync" "testing" "time" "go.ate.dev/substrate/internal/clustertrustbundle" "go.ate.dev/substrate/internal/installdefaults" - certsv1 "k8s.io/api/certificates/v1" - certsv1beta1 "k8s.io/api/certificates/v1beta1" + "go.ate.dev/substrate/internal/podidentity" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - "k8s.io/utils/ptr" ) // The router client's identity. @@ -54,13 +44,8 @@ func RouterClientSPIFFEID(serviceAccount string) string { return installdefaults.SPIFFEID(RouterClientNamespace, serviceAccount) } -const ( - // podIdentitySignerName issues the SPIFFE client certificates the router - // verifies. - podIdentitySignerName = "podidentity.podcert.ate.dev/identity" - // routerClientCertTimeout bounds the wait for the signer to issue. - routerClientCertTimeout = 2 * time.Minute -) +// routerClientCertTimeout bounds the wait for the signer to issue. +const routerClientCertTimeout = 2 * time.Minute var ( routerClientCertsMu sync.Mutex @@ -83,7 +68,7 @@ func RouterClientCertificate(t *testing.T, ctx context.Context, serviceAccount s if err := ensureRouterClientNamespace(ctx, clientset); err != nil { t.Fatalf("preparing namespace %s: %v", RouterClientNamespace, err) } - cert, err := requestPodIdentityCertificate(ctx, clientset, RouterClientNamespace, serviceAccount, routerClientCertTimeout) + cert, err := podidentity.RequestCertificate(ctx, clientset, "e2e", RouterClientNamespace, serviceAccount, routerClientCertTimeout) if err != nil { t.Fatalf("requesting a podidentity certificate for %s/%s: %v", RouterClientNamespace, serviceAccount, err) } @@ -91,179 +76,6 @@ func RouterClientCertificate(t *testing.T, ctx context.Context, serviceAccount s return cert } -// requestPodIdentityCertificate creates a PodCertificateRequest for -// serviceAccount in namespace, waits for the podidentity signer to issue it, -// and deletes it again. The key never leaves this process. -// -// The pod, ServiceAccount and node the request names do not exist: -// kube-apiserver only checks them for requests a kubelet creates, and the -// signer takes the identity from the request. -func requestPodIdentityCertificate(ctx context.Context, clientset kubernetes.Interface, namespace, serviceAccount string, timeout time.Duration) (tls.Certificate, error) { - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return tls.Certificate{}, fmt.Errorf("generating key: %w", err) - } - csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{}, key) - if err != nil { - return tls.Certificate{}, fmt.Errorf("creating stub CSR: %w", err) - } - - // Unique per call: suites run as concurrent processes and share the - // namespace. rand.Text is upper-case base32; object names are lower case. - name := "e2e-" + serviceAccount + "-" + strings.ToLower(rand.Text()[:8]) - pcr := &certsv1.PodCertificateRequest{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, - Spec: certsv1.PodCertificateRequestSpec{ - SignerName: podIdentitySignerName, - PodName: name, - PodUID: uuid.NewUUID(), - ServiceAccountName: serviceAccount, - ServiceAccountUID: uuid.NewUUID(), - NodeName: "e2e-test-process", - NodeUID: uuid.NewUUID(), - MaxExpirationSeconds: ptr.To(int32(24 * 60 * 60)), - StubPKCS10Request: csr, - }, - } - pcrs, err := podCertificateRequests(clientset, namespace) - if err != nil { - return tls.Certificate{}, err - } - if err := pcrs.create(ctx, pcr); err != nil { - return tls.Certificate{}, fmt.Errorf("creating PodCertificateRequest %s/%s: %w", namespace, name, err) - } - defer func() { - // A leftover request is harmless: kube-apiserver garbage collects - // old PodCertificateRequests. - delCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if err := pcrs.delete(delCtx, name); err != nil && !apierrors.IsNotFound(err) { - fmt.Fprintf(os.Stderr, "Failed to delete PodCertificateRequest %s/%s: %v\n", namespace, name, err) - } - }() - - var chainPEM string - err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, timeout, true, func(ctx context.Context) (bool, error) { - status, err := pcrs.status(ctx, name) - if err != nil { - return false, err - } - for _, c := range status.Conditions { - if c.Status != metav1.ConditionTrue { - continue - } - switch c.Type { - case certsv1.PodCertificateRequestConditionTypeIssued: - chainPEM = status.CertificateChain - return true, nil - case certsv1.PodCertificateRequestConditionTypeDenied, certsv1.PodCertificateRequestConditionTypeFailed: - return false, fmt.Errorf("%s: %s: %s", c.Type, c.Reason, c.Message) - } - } - return false, nil - }) - if err != nil { - return tls.Certificate{}, fmt.Errorf("waiting for PodCertificateRequest %s/%s to be issued: %w", namespace, name, err) - } - - cert := tls.Certificate{PrivateKey: key} - for rest := []byte(chainPEM); ; { - var block *pem.Block - block, rest = pem.Decode(rest) - if block == nil { - break - } - cert.Certificate = append(cert.Certificate, block.Bytes) - } - if len(cert.Certificate) == 0 { - return tls.Certificate{}, fmt.Errorf("PodCertificateRequest %s/%s was issued with no certificates", namespace, name) - } - leaf, err := x509.ParseCertificate(cert.Certificate[0]) - if err != nil { - return tls.Certificate{}, fmt.Errorf("parsing the issued leaf certificate: %w", err) - } - if !key.PublicKey.Equal(leaf.PublicKey) { - return tls.Certificate{}, fmt.Errorf("the issued certificate does not carry the requested public key") - } - if want := installdefaults.SPIFFEID(namespace, serviceAccount); len(leaf.URIs) != 1 || leaf.URIs[0].String() != want { - return tls.Certificate{}, fmt.Errorf("the podidentity signer issued a certificate with URI SANs %v, want exactly %s", leaf.URIs, want) - } - cert.Leaf = leaf - return cert, nil -} - -// pcrClient creates, reads and deletes PodCertificateRequests through -// whichever of certificates.k8s.io v1 and v1beta1 the cluster serves, -// preferring v1, as the podcertificate controller does. -type pcrClient struct { - clientset kubernetes.Interface - namespace string - v1 bool -} - -func podCertificateRequests(clientset kubernetes.Interface, namespace string) (*pcrClient, error) { - for _, version := range []string{"v1", "v1beta1"} { - resources, err := clientset.Discovery().ServerResourcesForGroupVersion("certificates.k8s.io/" + version) - if apierrors.IsNotFound(err) { - continue - } - if err != nil { - return nil, fmt.Errorf("discovering PodCertificateRequest %s: %w", version, err) - } - for _, r := range resources.APIResources { - if r.Name == "podcertificaterequests" { - return &pcrClient{clientset: clientset, namespace: namespace, v1: version == "v1"}, nil - } - } - } - return nil, fmt.Errorf("the cluster serves neither v1 nor v1beta1 PodCertificateRequest") -} - -func (c *pcrClient) create(ctx context.Context, pcr *certsv1.PodCertificateRequest) error { - if c.v1 { - _, err := c.clientset.CertificatesV1().PodCertificateRequests(c.namespace).Create(ctx, pcr, metav1.CreateOptions{}) - return err - } - beta := &certsv1beta1.PodCertificateRequest{ - ObjectMeta: pcr.ObjectMeta, - Spec: certsv1beta1.PodCertificateRequestSpec{ - SignerName: pcr.Spec.SignerName, - PodName: pcr.Spec.PodName, - PodUID: pcr.Spec.PodUID, - ServiceAccountName: pcr.Spec.ServiceAccountName, - ServiceAccountUID: pcr.Spec.ServiceAccountUID, - NodeName: pcr.Spec.NodeName, - NodeUID: pcr.Spec.NodeUID, - MaxExpirationSeconds: pcr.Spec.MaxExpirationSeconds, - StubPKCS10Request: pcr.Spec.StubPKCS10Request, - }, - } - _, err := c.clientset.CertificatesV1beta1().PodCertificateRequests(c.namespace).Create(ctx, beta, metav1.CreateOptions{}) - return err -} - -func (c *pcrClient) status(ctx context.Context, name string) (certsv1.PodCertificateRequestStatus, error) { - if c.v1 { - pcr, err := c.clientset.CertificatesV1().PodCertificateRequests(c.namespace).Get(ctx, name, metav1.GetOptions{}) - if err != nil { - return certsv1.PodCertificateRequestStatus{}, err - } - return pcr.Status, nil - } - pcr, err := c.clientset.CertificatesV1beta1().PodCertificateRequests(c.namespace).Get(ctx, name, metav1.GetOptions{}) - if err != nil { - return certsv1.PodCertificateRequestStatus{}, err - } - return certsv1.PodCertificateRequestStatus(pcr.Status), nil -} - -func (c *pcrClient) delete(ctx context.Context, name string) error { - if c.v1 { - return c.clientset.CertificatesV1().PodCertificateRequests(c.namespace).Delete(ctx, name, metav1.DeleteOptions{}) - } - return c.clientset.CertificatesV1beta1().PodCertificateRequests(c.namespace).Delete(ctx, name, metav1.DeleteOptions{}) -} - // ensureRouterClientNamespace creates RouterClientNamespace, leaving it in // place if it already exists. The namespace carries NamespaceLabel, so // hack/cleanup-e2e.sh removes it with the rest. diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 460db84990..f5b856222c 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -15,7 +15,6 @@ package e2e import ( - "bufio" "bytes" "context" "crypto/tls" @@ -23,9 +22,6 @@ import ( "io" "net" "net/http" - "net/url" - "strconv" - "strings" "testing" "time" @@ -193,79 +189,5 @@ func (c *RouterClient) ConnectWithHeaders(ctx context.Context, actorRef resource if err != nil { return nil, err } - conn, err := c.connect(ctx, addr, actorRef, port, header) - if err != nil { - stop() - return nil, err - } - conn.stop = stop - return conn, nil -} - -func (c *RouterClient) connect(ctx context.Context, addr string, actorRef resources.ActorRef, port int, header http.Header) (*bufferedConn, error) { - tcpConn, err := (&net.Dialer{}).DialContext(ctx, "tcp", addr) - if err != nil { - return nil, fmt.Errorf("connecting to router's CONNECT listener: %w", err) - } - rawConn := tls.Client(tcpConn, c.tlsConfig.Clone()) - if err := rawConn.HandshakeContext(ctx); err != nil { - _ = tcpConn.Close() - return nil, fmt.Errorf("TLS handshake with router's CONNECT listener: %w", err) - } - - destination := net.JoinHostPort(actorRef.Name, strconv.Itoa(port)) - req := &http.Request{ - Method: http.MethodConnect, - URL: &url.URL{Host: destination}, - Host: destination, - Header: header, - } - if err := req.Write(rawConn); err != nil { - _ = rawConn.Close() - return nil, fmt.Errorf("writing CONNECT request: %w", err) - } - - reader := bufio.NewReader(rawConn) - resp, err := http.ReadResponse(reader, req) - if err != nil { - _ = rawConn.Close() - return nil, fmt.Errorf("reading CONNECT response: %w", err) - } - if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) - _ = resp.Body.Close() - _ = rawConn.Close() - message := strings.TrimSpace(string(body)) - if message == "" { - message = resp.Status - } - return nil, fmt.Errorf("router rejected CONNECT to %s with %s: %s", destination, resp.Status, message) - } - - // http.ReadResponse may have buffered bytes past the header boundary into - // reader; wrap the connection so a caller's Read sees them instead of - // losing them to reader's own buffer. - return &bufferedConn{Conn: rawConn, reader: reader}, nil -} - -// bufferedConn recovers bytes http.ReadResponse buffered past the header -// boundary, mirroring internal/atunnel/egress/client.go's identical need on the -// egress CONNECT client. -type bufferedConn struct { - net.Conn - reader *bufio.Reader - stop func() -} - -func (c *bufferedConn) Read(p []byte) (int, error) { - return c.reader.Read(p) -} - -func (c *bufferedConn) Close() error { - err := c.Conn.Close() - if c.stop != nil { - c.stop() - c.stop = nil - } - return err + return ateclient.ConnectActorWithHeaders(ctx, addr, c.tlsConfig, actorRef, port, header, stop) } diff --git a/internal/e2e/router_client_test.go b/internal/e2e/router_client_test.go index c8c8753759..2bfd0bf11d 100644 --- a/internal/e2e/router_client_test.go +++ b/internal/e2e/router_client_test.go @@ -30,6 +30,7 @@ import ( "testing" "time" + "go.ate.dev/substrate/internal/ateclient" "go.ate.dev/substrate/internal/atenet" "go.ate.dev/substrate/internal/resources" ) @@ -83,7 +84,8 @@ func (f testRoundTripper) RoundTrip(request *http.Request) (*http.Response, erro } // Connect has to complete a TLS handshake, presenting the client certificate, -// before it sends the CONNECT. +// before it sends the CONNECT. The httptest server requires and verifies the +// client certificate over TLS, so a plaintext dial fails the handshake. func TestRouterClientConnectTLS(t *testing.T) { clientCert := selfSignedClientCert(t, "spiffe://cluster.local/ns/test/sa/client") clientCAs := x509.NewCertPool() @@ -122,18 +124,14 @@ func TestRouterClientConnectTLS(t *testing.T) { ServerName: "example.com", } tlsConfig.RootCAs.AddCert(server.Certificate()) - client := &RouterClient{tlsConfig: tlsConfig} - conn, err := client.connect(context.Background(), server.Listener.Addr().String(), resources.ActorRef{Atespace: "demo", Name: "fetcher"}, 9090, http.Header{ + conn, err := ateclient.ConnectActorWithHeaders(context.Background(), server.Listener.Addr().String(), tlsConfig, resources.ActorRef{Atespace: "demo", Name: "fetcher"}, 9090, http.Header{ atenet.TargetActorHeader: []string{"demo/fetcher"}, - }) + }, nil) if err != nil { t.Fatalf("connect: %v", err) } defer conn.Close() - if _, ok := conn.Conn.(*tls.Conn); !ok { - t.Errorf("connect returned a %T tunnel, want one over TLS", conn.Conn) - } got, err := io.ReadAll(conn) if err != nil { t.Fatalf("reading the tunnel: %v", err) diff --git a/internal/podidentity/podidentity.go b/internal/podidentity/podidentity.go new file mode 100644 index 0000000000..889c63a6bd --- /dev/null +++ b/internal/podidentity/podidentity.go @@ -0,0 +1,216 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package podidentity requests certificates from substrate's podidentity +// signer for processes that do not run in a pod. +package podidentity + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "time" + + certsv1 "k8s.io/api/certificates/v1" + certsv1beta1 "k8s.io/api/certificates/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" +) + +// SignerName is the podidentity signer, which issues the SPIFFE certificates +// the ingress router verifies. +const SignerName = "podidentity.podcert.ate.dev/identity" + +// RequestCertificate creates a PodCertificateRequest for serviceAccount in +// namespace, waits for the podidentity signer to issue it, and deletes it +// again. The key never leaves this process. +// +// The pod, ServiceAccount and node the request names do not exist: +// kube-apiserver only checks them for requests a kubelet creates, and the +// signer takes the identity from the request. requester names the node and +// prefixes the request's name, so the request shows who made it. +func RequestCertificate(ctx context.Context, clientset kubernetes.Interface, requester, namespace, serviceAccount string, timeout time.Duration) (tls.Certificate, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generating key: %w", err) + } + csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{}, key) + if err != nil { + return tls.Certificate{}, fmt.Errorf("creating stub CSR: %w", err) + } + + pcr := &certsv1.PodCertificateRequest{ + ObjectMeta: metav1.ObjectMeta{GenerateName: requester + "-", Namespace: namespace}, + Spec: certsv1.PodCertificateRequestSpec{ + SignerName: SignerName, + PodName: requester, + PodUID: uuid.NewUUID(), + ServiceAccountName: serviceAccount, + ServiceAccountUID: uuid.NewUUID(), + NodeName: types.NodeName(requester), + NodeUID: uuid.NewUUID(), + MaxExpirationSeconds: ptr.To(int32(24 * 60 * 60)), + StubPKCS10Request: csr, + }, + } + pcrs, err := podCertificateRequests(clientset, namespace) + if err != nil { + return tls.Certificate{}, err + } + name, err := pcrs.create(ctx, pcr) + if err != nil { + return tls.Certificate{}, fmt.Errorf("creating PodCertificateRequest in %s: %w", namespace, err) + } + defer func() { + // A leftover request is harmless: kube-apiserver garbage collects + // old PodCertificateRequests. + delCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _ = pcrs.delete(delCtx, name) + }() + + var chainPEM string + err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, timeout, true, func(ctx context.Context) (bool, error) { + status, err := pcrs.status(ctx, name) + if err != nil { + return false, err + } + for _, c := range status.Conditions { + if c.Status != metav1.ConditionTrue { + continue + } + switch c.Type { + case certsv1.PodCertificateRequestConditionTypeIssued: + chainPEM = status.CertificateChain + return true, nil + case certsv1.PodCertificateRequestConditionTypeDenied, certsv1.PodCertificateRequestConditionTypeFailed: + return false, fmt.Errorf("%s: %s: %s", c.Type, c.Reason, c.Message) + } + } + return false, nil + }) + if err != nil { + return tls.Certificate{}, fmt.Errorf("waiting for PodCertificateRequest %s/%s to be issued: %w", namespace, name, err) + } + + cert := tls.Certificate{PrivateKey: key} + for rest := []byte(chainPEM); ; { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + cert.Certificate = append(cert.Certificate, block.Bytes) + } + if len(cert.Certificate) == 0 { + return tls.Certificate{}, fmt.Errorf("PodCertificateRequest %s/%s was issued with no certificates", namespace, name) + } + leaf, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return tls.Certificate{}, fmt.Errorf("parsing the issued leaf certificate: %w", err) + } + if !key.PublicKey.Equal(leaf.PublicKey) { + return tls.Certificate{}, fmt.Errorf("the issued certificate does not carry the requested public key") + } + cert.Leaf = leaf + return cert, nil +} + +// pcrClient creates, reads and deletes PodCertificateRequests through +// whichever of certificates.k8s.io v1 and v1beta1 the cluster serves, +// preferring v1, as the podcertificate controller does. +type pcrClient struct { + clientset kubernetes.Interface + namespace string + v1 bool +} + +func podCertificateRequests(clientset kubernetes.Interface, namespace string) (*pcrClient, error) { + for _, version := range []string{"v1", "v1beta1"} { + resources, err := clientset.Discovery().ServerResourcesForGroupVersion("certificates.k8s.io/" + version) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return nil, fmt.Errorf("discovering PodCertificateRequest %s: %w", version, err) + } + for _, r := range resources.APIResources { + if r.Name == "podcertificaterequests" { + return &pcrClient{clientset: clientset, namespace: namespace, v1: version == "v1"}, nil + } + } + } + return nil, fmt.Errorf("the cluster serves neither v1 nor v1beta1 PodCertificateRequest") +} + +func (c *pcrClient) create(ctx context.Context, pcr *certsv1.PodCertificateRequest) (string, error) { + if c.v1 { + created, err := c.clientset.CertificatesV1().PodCertificateRequests(c.namespace).Create(ctx, pcr, metav1.CreateOptions{}) + if err != nil { + return "", err + } + return created.Name, nil + } + beta := &certsv1beta1.PodCertificateRequest{ + ObjectMeta: pcr.ObjectMeta, + Spec: certsv1beta1.PodCertificateRequestSpec{ + SignerName: pcr.Spec.SignerName, + PodName: pcr.Spec.PodName, + PodUID: pcr.Spec.PodUID, + ServiceAccountName: pcr.Spec.ServiceAccountName, + ServiceAccountUID: pcr.Spec.ServiceAccountUID, + NodeName: pcr.Spec.NodeName, + NodeUID: pcr.Spec.NodeUID, + MaxExpirationSeconds: pcr.Spec.MaxExpirationSeconds, + StubPKCS10Request: pcr.Spec.StubPKCS10Request, + }, + } + created, err := c.clientset.CertificatesV1beta1().PodCertificateRequests(c.namespace).Create(ctx, beta, metav1.CreateOptions{}) + if err != nil { + return "", err + } + return created.Name, nil +} + +func (c *pcrClient) status(ctx context.Context, name string) (certsv1.PodCertificateRequestStatus, error) { + if c.v1 { + pcr, err := c.clientset.CertificatesV1().PodCertificateRequests(c.namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return certsv1.PodCertificateRequestStatus{}, err + } + return pcr.Status, nil + } + pcr, err := c.clientset.CertificatesV1beta1().PodCertificateRequests(c.namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return certsv1.PodCertificateRequestStatus{}, err + } + return certsv1.PodCertificateRequestStatus(pcr.Status), nil +} + +func (c *pcrClient) delete(ctx context.Context, name string) error { + if c.v1 { + return c.clientset.CertificatesV1().PodCertificateRequests(c.namespace).Delete(ctx, name, metav1.DeleteOptions{}) + } + return c.clientset.CertificatesV1beta1().PodCertificateRequests(c.namespace).Delete(ctx, name, metav1.DeleteOptions{}) +} diff --git a/internal/e2e/ingressauth_test.go b/internal/podidentity/podidentity_test.go similarity index 76% rename from internal/e2e/ingressauth_test.go rename to internal/podidentity/podidentity_test.go index d97dc14efe..53c72b58d0 100644 --- a/internal/e2e/ingressauth_test.go +++ b/internal/podidentity/podidentity_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package e2e +package podidentity import ( "context" @@ -22,12 +22,10 @@ import ( "crypto/x509" "encoding/pem" "math/big" - "net/url" "strings" "testing" "time" - "go.ate.dev/substrate/internal/installdefaults" certsv1 "k8s.io/api/certificates/v1" certsv1beta1 "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,13 +34,17 @@ import ( k8stesting "k8s.io/client-go/testing" ) +const ( + testNamespace = "ns" + testServiceAccount = "client" +) + // fakePodIdentitySigner answers PodCertificateRequests the way the podidentity // signer does, signing the stub CSR's key with a throwaway CA. type fakePodIdentitySigner struct { t *testing.T ca *x509.Certificate caKey *ecdsa.PrivateKey - uri func(spec certsv1.PodCertificateRequestSpec) string denied bool // got is the spec of the last request created. got certsv1.PodCertificateRequestSpec @@ -72,10 +74,6 @@ func newFakePodIdentitySigner(t *testing.T) *fakePodIdentitySigner { } return &fakePodIdentitySigner{ t: t, ca: ca, caKey: caKey, - uri: func(spec certsv1.PodCertificateRequestSpec) string { - // The namespace is not in the spec; the tests use one namespace. - return installdefaults.SPIFFEID(RouterClientNamespace, spec.ServiceAccountName) - }, } } @@ -92,15 +90,10 @@ func (s *fakePodIdentitySigner) status(spec certsv1.PodCertificateRequestSpec) c if err != nil { s.t.Fatalf("parsing the stub CSR: %v", err) } - uri, err := url.Parse(s.uri(spec)) - if err != nil { - s.t.Fatal(err) - } der, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ SerialNumber: big.NewInt(2), NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), - URIs: []*url.URL{uri}, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, }, s.ca, csr.PublicKey, s.caKey) if err != nil { @@ -126,8 +119,10 @@ func (s *fakePodIdentitySigner) install(kc *fake.Clientset, version string) { kc.PrependReactor("create", "podcertificaterequests", func(action k8stesting.Action) (bool, runtime.Object, error) { switch pcr := action.(k8stesting.CreateAction).GetObject().(type) { case *certsv1.PodCertificateRequest: + pcr.Name = pcr.GenerateName + "abcde" pcr.Status = s.status(pcr.Spec) case *certsv1beta1.PodCertificateRequest: + pcr.Name = pcr.GenerateName + "abcde" pcr.Status = certsv1beta1.PodCertificateRequestStatus(s.status(certsv1.PodCertificateRequestSpec{ SignerName: pcr.Spec.SignerName, ServiceAccountName: pcr.Spec.ServiceAccountName, @@ -140,26 +135,26 @@ func (s *fakePodIdentitySigner) install(kc *fake.Clientset, version string) { }) } -func TestRequestPodIdentityCertificate(t *testing.T) { +func TestRequestCertificate(t *testing.T) { for _, version := range []string{"v1", "v1beta1"} { t.Run(version, func(t *testing.T) { kc := fake.NewSimpleClientset() signer := newFakePodIdentitySigner(t) signer.install(kc, version) - cert, err := requestPodIdentityCertificate(context.Background(), kc, RouterClientNamespace, "router-client", time.Second) + cert, err := RequestCertificate(context.Background(), kc, "test", testNamespace, testServiceAccount, time.Second) if err != nil { - t.Fatalf("requestPodIdentityCertificate: %v", err) + t.Fatalf("RequestCertificate: %v", err) } - if signer.got.SignerName != podIdentitySignerName { - t.Errorf("signerName = %q, want %q", signer.got.SignerName, podIdentitySignerName) + if signer.got.SignerName != SignerName { + t.Errorf("signerName = %q, want %q", signer.got.SignerName, SignerName) } - if signer.got.ServiceAccountName != "router-client" { - t.Errorf("serviceAccountName = %q, want router-client", signer.got.ServiceAccountName) + if signer.got.ServiceAccountName != testServiceAccount { + t.Errorf("serviceAccountName = %q, want %q", signer.got.ServiceAccountName, testServiceAccount) } - if want := RouterClientSPIFFEID("router-client"); cert.Leaf == nil || len(cert.Leaf.URIs) != 1 || cert.Leaf.URIs[0].String() != want { - t.Fatalf("leaf URIs = %v, want [%s]", cert.Leaf.URIs, want) + if cert.Leaf == nil { + t.Fatal("cert.Leaf is nil") } key, ok := cert.PrivateKey.(*ecdsa.PrivateKey) if !ok || !key.PublicKey.Equal(cert.Leaf.PublicKey) { @@ -169,13 +164,13 @@ func TestRequestPodIdentityCertificate(t *testing.T) { // The request is deleted once the certificate is in hand. var remaining int if version == "v1" { - list, err := kc.CertificatesV1().PodCertificateRequests(RouterClientNamespace).List(context.Background(), metav1.ListOptions{}) + list, err := kc.CertificatesV1().PodCertificateRequests(testNamespace).List(context.Background(), metav1.ListOptions{}) if err != nil { t.Fatal(err) } remaining = len(list.Items) } else { - list, err := kc.CertificatesV1beta1().PodCertificateRequests(RouterClientNamespace).List(context.Background(), metav1.ListOptions{}) + list, err := kc.CertificatesV1beta1().PodCertificateRequests(testNamespace).List(context.Background(), metav1.ListOptions{}) if err != nil { t.Fatal(err) } @@ -188,7 +183,7 @@ func TestRequestPodIdentityCertificate(t *testing.T) { } } -func TestRequestPodIdentityCertificateErrors(t *testing.T) { +func TestRequestCertificateErrors(t *testing.T) { for _, tc := range []struct { name string setup func(t *testing.T, kc *fake.Clientset) @@ -203,17 +198,6 @@ func TestRequestPodIdentityCertificateErrors(t *testing.T) { }, wantErr: "Denied", }, - { - name: "wrong SPIFFE ID", - setup: func(t *testing.T, kc *fake.Clientset) { - s := newFakePodIdentitySigner(t) - s.uri = func(certsv1.PodCertificateRequestSpec) string { - return "spiffe://cluster.local/ns/other/sa/router-client" - } - s.install(kc, "v1") - }, - wantErr: "URI SANs", - }, { name: "never issued", setup: func(t *testing.T, kc *fake.Clientset) { @@ -233,9 +217,9 @@ func TestRequestPodIdentityCertificateErrors(t *testing.T) { t.Run(tc.name, func(t *testing.T) { kc := fake.NewSimpleClientset() tc.setup(t, kc) - _, err := requestPodIdentityCertificate(context.Background(), kc, RouterClientNamespace, "router-client", 100*time.Millisecond) + _, err := RequestCertificate(context.Background(), kc, "test", testNamespace, testServiceAccount, 100*time.Millisecond) if err == nil || !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("requestPodIdentityCertificate error = %v, want one containing %q", err, tc.wantErr) + t.Fatalf("RequestCertificate error = %v, want one containing %q", err, tc.wantErr) } }) } diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index d9185f591c..b9ac8d4e0b 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -227,10 +227,10 @@ spec: # # The allowed SPIFFE IDs are the install's choice, so this manifest # carries none: cmd/ate-setup appends --ingress-allowed-spiffe-ids from - # its own --ingress-allowed-spiffe-ids (ATE_INGRESS_ALLOWED_SPIFFE_IDS), - # and the router refuses to start without it. + # its own --ingress-allowed-spiffe-ids (ATE_INGRESS_ALLOWED_SPIFFE_IDS). + # Without one the router serves no traffic. - "--ingress-client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem" - # - "--ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-e2e-router-client/sa/router-client" + # - "--ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-system/sa/atenet-client" # Client auth to ateapi (mtls): verify the serving cert against the # servicedns trust bundle and present the podidentity client cert. - "--ateapi-address=k8s:///api.ate-system.svc:443" From 6aa6a7502d2994578070f8f629ca172f7bbc11fa Mon Sep 17 00:00:00 2001 From: nybidari <59618317+nybidari@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:35:53 +0000 Subject: [PATCH 23/53] Update gVisor to release 20261026.0 (#3364) gvisor-default gains v6, release 20261026.0, as its default version. v5 stays enabled, so actors with memory snapshots taken on v5 still restore with their memory. In a sandbox which ateworker-gvisor runs, the new release's restore picks each container's real init process. The old one could take an orphan, so signals reached the wrong process and runsc reported a running container as stopped. The new gVisor release includes some reliability fixes such as: - Fix nil pointer crash in restoreDone on the sandbox restore path. - Fix no-root-container-mode sandboxes: shared PID namespace could panic the Sentry. - Fix kernel state dependency-cycle which could panic or recurse forever. AI use: AI-assisted, I reviewed and tested it. - [x] Tests pass - [x] Docs updated Signed-off-by: Nayana Bidari --- docs/api-guide.md | 12 +++++----- .../ate-install/sandboxconfig-gvisor.yaml | 23 ++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/api-guide.md b/docs/api-guide.md index fdf3292b57..bd45a67223 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -561,19 +561,19 @@ metadata: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - defaultVersion: v5 + defaultVersion: v6 versions: - - name: v5 + - name: v6 state: Enabled assets: amd64: gvisor: - url: "gs://gvisor/releases/release/20261019.0/x86_64/gvisor.tar.zstd" - sha256: "8637abfb50f81552af25ca08f7172589c09292a7c881efe8d9e09a9f3b6bec7b" + url: "gs://gvisor/releases/release/20261026.0/x86_64/gvisor.tar.zstd" + sha256: "708afc62f2ee82d19930c703e85f9dab4f27a78c0aec40c0f55eb1040107f1a7" arm64: gvisor: - url: "gs://gvisor/releases/release/20261019.0/aarch64/gvisor.tar.zstd" - sha256: "88556c21468a066d872a055425958c83fcbafc79c6477a195b18070c77f856d0" + url: "gs://gvisor/releases/release/20261026.0/aarch64/gvisor.tar.zstd" + sha256: "6257e49955f13e6caceb2c4181c5dc5406c460e805940c944e67ad65553f5b02" ``` ### MicroVM SandboxConfig diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index 44b81a697d..a444c303a6 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -31,21 +31,22 @@ metadata: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - defaultVersion: v5 + defaultVersion: v6 versions: - # gVisor release 20261019.0, the first tagged release with - # google/gvisor@b79256080dec (save and restore multiple filesystem checkpoint - # bundles). ateworker-gvisor uses it to save container rootfs overlays - # (rootfs/) and durable-dir volumes (fs/) as separate bundles, and to run the - # sandbox with no root container (google/gvisor#15551). - - name: v5 + # gVisor release 20261026.0. ateworker-gvisor saves container rootfs overlays + # (rootfs/) and durable-dir volumes (fs/) as separate checkpoint bundles. That + # needs google/gvisor@b79256080dec, first in tagged release 20261019.0. + # ateworker-gvisor also runs the sandbox with no root container + # (google/gvisor#15551). This release's restore picks each container's init + # process correctly in such a sandbox (google/gvisor@29b97c66d945). + - name: v6 state: Enabled assets: amd64: gvisor: - url: "gs://gvisor/releases/release/20261019.0/x86_64/gvisor.tar.zstd" - sha256: "8637abfb50f81552af25ca08f7172589c09292a7c881efe8d9e09a9f3b6bec7b" + url: "gs://gvisor/releases/release/20261026.0/x86_64/gvisor.tar.zstd" + sha256: "708afc62f2ee82d19930c703e85f9dab4f27a78c0aec40c0f55eb1040107f1a7" arm64: gvisor: - url: "gs://gvisor/releases/release/20261019.0/aarch64/gvisor.tar.zstd" - sha256: "88556c21468a066d872a055425958c83fcbafc79c6477a195b18070c77f856d0" + url: "gs://gvisor/releases/release/20261026.0/aarch64/gvisor.tar.zstd" + sha256: "6257e49955f13e6caceb2c4181c5dc5406c460e805940c944e67ad65553f5b02" From 006b0af52b3dc7523e80d31176e8a9609a6cc574 Mon Sep 17 00:00:00 2001 From: Luiz Oliveira Date: Wed, 28 Oct 2026 02:22:45 +0000 Subject: [PATCH 24/53] Update a deleting tag returns FAILED_PRECONDITION (#3340) #3333 Signed-off-by: Luiz Oliveira --- .../controlapi/functionaltest/tag_test.go | 25 +++++++++++++++++++ cmd/ateapi/internal/controlapi/tag.go | 18 ++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go index 2091cf2535..fece94e98e 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go @@ -16,6 +16,7 @@ package functionaltest import ( "context" + "errors" "fmt" "testing" @@ -362,6 +363,30 @@ func TestUpdateTag_NotFound(t *testing.T) { assertGrpcError(t, err, codes.NotFound, "Tag test-atespace/does-not-exist not found") } +// TestUpdateTag_DeletingTag verifies a tag left in place by a failed delete +// can no longer be updated. +func TestUpdateTag_DeletingTag(t *testing.T) { + ns := namespaceForTest("ns-update-tag-deleting") + tc := setupTest(t, ns) + defer tc.cleanup() + ctx := context.Background() + createTemplate(t, tc, ns) + + tagRef := resources.TagRefFromTag(seedTag(t, tc, "actor-1", "v1")) + tc.objectStore.OnList = func(string, string) error { return errors.New("injected list failure") } + if _, err := tc.client.DeleteTag(ctx, &ateapi.DeleteTagRequest{Tag: tagRef.ToObjectRef()}); err == nil { + t.Fatal("DeleteTag succeeded despite the failing snapshot cleanup") + } + + marked, err := tc.client.GetTag(ctx, &ateapi.GetTagRequest{Tag: tagRef.ToObjectRef()}) + if err != nil { + t.Fatalf("GetTag after the failed delete: %v", err) + } + marked.Scope = ateapi.TagScope_TAG_SCOPE_PUBLISHED + _, err = tc.client.UpdateTag(ctx, &ateapi.UpdateTagRequest{Tag: marked}) + assertGrpcError(t, err, codes.FailedPrecondition, "Tag "+tagRef.String()+" is being deleted") +} + func TestDeleteTag(t *testing.T) { ns := namespaceForTest("ns-delete-tag") tc := setupTest(t, ns) diff --git a/cmd/ateapi/internal/controlapi/tag.go b/cmd/ateapi/internal/controlapi/tag.go index b53a44364b..6fdc1fa015 100644 --- a/cmd/ateapi/internal/controlapi/tag.go +++ b/cmd/ateapi/internal/controlapi/tag.go @@ -100,10 +100,14 @@ func (s *ServiceImpl) ListTags(ctx context.Context, atespace string, opts store. return s.store.ListTags(ctx, atespace, opts) } -// errTagPending is what the update's mutate closure returns when the stored -// tag's create never finished, so the caller can tell it apart from a store -// failure and answer FAILED_PRECONDITION. -var errTagPending = errors.New("tag is still being created") +// errTagPending and errTagDeleting are what the update's mutate closure +// returns when the stored tag's create never finished or its delete has +// started, so the caller can tell them apart from a store failure and answer +// FAILED_PRECONDITION. +var ( + errTagPending = errors.New("tag is still being created") + errTagDeleting = errors.New("tag is being deleted") +) func (s *RPCService) UpdateTag(ctx context.Context, req *ateapi.UpdateTagRequest) (*ateapi.Tag, error) { // First scrub any fields that users are not allowed to set. @@ -120,6 +124,9 @@ func (s *RPCService) UpdateTag(ctx context.Context, req *ateapi.UpdateTagRequest tagRef := resources.TagRefFromTag(in) storedTag, err := s.impl.UpdateTag(ctx, tagRef, store.PreconditionFrom(in), func(toUpdate *ateapi.Tag) error { + if toUpdate.GetMetadata().GetDeleteTime() != nil { + return errTagDeleting + } // A tag whose create never finished names a partial copy. Publishing it // — or changing its scope at all — would hand out content that is still // being written, or may never be. @@ -146,6 +153,9 @@ func (s *RPCService) UpdateTag(ctx context.Context, req *ateapi.UpdateTagRequest if errors.Is(err, errTagPending) { return nil, apierror.FailedPrecondition("Tag %s/%s is still being created", tagRef.Atespace, tagRef.Name) } + if errors.Is(err, errTagDeleting) { + return nil, apierror.FailedPrecondition("Tag %s/%s is being deleted", tagRef.Atespace, tagRef.Name) + } if errors.Is(err, store.ErrImmutableField) { return nil, apierror.InvalidArgument("while updating tag %s/%s: %v", tagRef.Atespace, tagRef.Name, err) } From 7a19374389271d3c5a60c51b107b176d12248616 Mon Sep 17 00:00:00 2001 From: nybidari <59618317+nybidari@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:11:47 +0000 Subject: [PATCH 25/53] e2e: test RevertActor across multiple durable snapshot generations (#3338) Verify that RevertActor returns an actor to its most recent completed durable snapshot from RUNNING, paused SUSPENDED after background upload, and CRASHED states, and reverts to the template golden snapshot when the actor has no snapshot of its own. Fixes part of #3147 AI use: AI-assisted, I reviewed and tested it. No breaking change - [x] Tests pass - [x] Docs updated Signed-off-by: Nayana Bidari --- internal/e2e/suites/demo/demo_test.go | 265 +++++++++++++++++++++++++- 1 file changed, 260 insertions(+), 5 deletions(-) diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 1da963a945..0a30906e50 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -1789,13 +1789,33 @@ func TestRevertCrashedActor(t *testing.T) { } validateCounterResponse(t, resp, "before snapshot", 1, 1) + suspended1, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapi.SuspendActorRequest{Actor: actorRef, Options: &ateapi.SuspendOptions{Mode: ateapi.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) + if err != nil { + t.Fatalf("failed to suspend Actor (generation 1): %v", err) + } + gen1URI := durableSnapshotURI(suspended1.GetActor().GetStatus()) + if gen1URI == "" { + t.Fatal("suspend (generation 1) wrote no external snapshot") + } + + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapi.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("failed to resume Actor after generation 1 suspend: %v", err) + } + waitForActorState(ctx, t, clients, actorName, ateapi.ActorState_ACTOR_STATE_RUNNING) + + resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) + if err != nil { + t.Fatalf("failed to call actor before generation 2 snapshot: %v", err) + } + validateCounterResponse(t, resp, "before generation 2 snapshot", 2, 2) + suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapi.SuspendActorRequest{Actor: actorRef, Options: &ateapi.SuspendOptions{Mode: ateapi.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { - t.Fatalf("failed to suspend Actor: %v", err) + t.Fatalf("failed to suspend Actor (generation 2): %v", err) } snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) - if snapshotURI == "" { - t.Fatal("suspend wrote no external snapshot") + if snapshotURI == "" || snapshotURI == gen1URI { + t.Fatalf("generation 2 snapshot URI = %q, want non-empty and distinct from generation 1 %q", snapshotURI, gen1URI) } if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapi.ResumeActorRequest{Actor: actorRef}); err != nil { @@ -1807,7 +1827,7 @@ func TestRevertCrashedActor(t *testing.T) { if err != nil { t.Fatalf("failed to call actor before crash: %v", err) } - validateCounterResponse(t, resp, "before crash", 2, 2) + validateCounterResponse(t, resp, "before crash", 3, 3) actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapi.GetActorRequest{Actor: actorRef}) if err != nil { @@ -1862,5 +1882,240 @@ func TestRevertCrashedActor(t *testing.T) { if err != nil { t.Fatalf("failed to call actor after recovery: %v", err) } - validateCounterResponse(t, resp, "after recovery from CRASHED", 2, 2) + validateCounterResponse(t, resp, "after recovery from CRASHED", 3, 3) +} + +// TestRevertActorToMostRecentDurableSnapshot verifies that RevertActor returns +// the actor to its most recent completed durable snapshot across origin states +// and multiple snapshot generations. +func TestRevertActorToMostRecentDurableSnapshot(t *testing.T) { + t.Parallel() + nsObj := e2e.CreateNamespace(t) + ctx := context.Background() + clients := e2e.GetClients() + + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) + if err != nil { + t.Fatalf("failed to initialize ActorTemplate: %v", err) + } + + tests := []struct { + name string + actorPrefix string + // setup drives the actor into the state from which RevertActor is + // called, returning the durable snapshot URI and counter value the + // revert must preserve. + setup func(ctx context.Context, t *testing.T, h *revertHarness) (wantURI string, wantCounter int) + }{ + { + name: "FromRunningBeforeActorSnapshotRevertsToGolden", + actorPrefix: "revert-to-golden", + setup: func(ctx context.Context, t *testing.T, h *revertHarness) (string, int) { + if h.createdURI == "" { + t.Fatal("new actor has no golden snapshot") + } + h.resumeAndIncrement(ctx, 1) + return h.createdURI, 0 + }, + }, + { + name: "FromRunningAfterMultipleDurableSnapshots", + actorPrefix: "revert-multi-running", + setup: func(ctx context.Context, t *testing.T, h *revertHarness) (string, int) { + // Generation 1: synchronous durable snapshot at count=1. + h.resumeAndIncrement(ctx, 1) + gen1URI := h.suspendSync(ctx) + + // Generation 2: asynchronous suspend at count=2, then wait for + // atelet's background upload to complete so generation 2 becomes + // the most recent durable snapshot. + h.resumeAndIncrement(ctx, 2) + gen2URI := h.suspendAsyncAndAwaitUpload(ctx) + if gen2URI == gen1URI { + t.Fatalf("generation 2 durable URI %q did not advance from generation 1", gen2URI) + } + + // Mutate in a third sprint (count=3) that must be discarded. + h.resumeAndIncrement(ctx, 3) + return gen2URI, 2 + }, + }, + { + name: "FromPausedAfterBackgroundUploadCompleted", + actorPrefix: "revert-paused-uploaded", + setup: func(ctx context.Context, t *testing.T, h *revertHarness) (string, int) { + // Generation 1 durable at count=1. + h.resumeAndIncrement(ctx, 1) + _ = h.suspendSync(ctx) + + // Generation 2: pause at count=2 and wait for the upload. The + // local checkpoint stays on the node, so the revert starts from + // a paused actor whose latest snapshot is already durable. + h.resumeAndIncrement(ctx, 2) + gen2URI := h.suspendAsyncAndAwaitUpload(ctx) + waitForActorPaused(ctx, t, h.clients, h.name) + return gen2URI, 2 + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + h := newRevertHarness(ctx, t, clients, nsObj, at, tc.actorPrefix) + wantURI, wantCounter := tc.setup(ctx, t, h) + + reverted := h.revertAndAssert(ctx, wantURI) + + // Calling RevertActor again when already at the durable snapshot + // must fail with FailedPrecondition. + if _, err := clients.SubstrateAPI.RevertActor(ctx, &ateapi.RevertActorRequest{Actor: h.ref}); status.Code(err) != codes.FailedPrecondition { + t.Fatalf("RevertActor on already-reverted actor %v: got %v, want FailedPrecondition", reverted.GetMetadata().GetName(), err) + } + + // Resuming and calling the actor increments from wantCounter to + // wantCounter+1, proving the most recent durable snapshot won. + h.resumeAndIncrement(ctx, wantCounter+1) + }) + } +} + +type revertHarness struct { + t *testing.T + clients *e2e.Clients + name string + ref *ateapi.ObjectRef + // createdURI is the durable snapshot the actor was created from: the + // template's golden snapshot. + createdURI string +} + +func newRevertHarness(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, at *ateapi.ActorTemplate, actorPrefix string) *revertHarness { + t.Helper() + actorName := actorPrefix + "-" + nsObj.Name + actorRef := &ateapi.ObjectRef{Atespace: demoAtespace, Name: actorName} + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + _, _ = clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapi.DeleteActorRequest{Actor: actorRef}) + }) + + created, err := clients.SubstrateAPI.CreateActor(ctx, &ateapi.CreateActorRequest{Actor: &ateapi.Actor{ + Metadata: &ateapi.ResourceMetadata{Atespace: demoAtespace, Name: actorName}, + ActorTemplate: e2e.TemplateRef(at), + }}) + if err != nil { + t.Fatalf("CreateActor %q: %v", actorName, err) + } + waitForActorState(ctx, t, clients, actorName, ateapi.ActorState_ACTOR_STATE_SUSPENDED) + + return &revertHarness{ + t: t, + clients: clients, + name: actorName, + ref: actorRef, + createdURI: durableSnapshotURI(created.GetStatus()), + } +} + +func (h *revertHarness) resumeAndIncrement(ctx context.Context, wantCount int) { + h.t.Helper() + if _, err := e2e.ResumeActorAwaitCapacity(h.t, ctx, h.clients, &ateapi.ResumeActorRequest{Actor: h.ref}); err != nil { + h.t.Fatalf("ResumeActor %q: %v", h.name, err) + } + waitForActorState(ctx, h.t, h.clients, h.name, ateapi.ActorState_ACTOR_STATE_RUNNING) + + resp, err := callActor(h.t, resources.ActorRefFromObjectRef(h.ref)) + if err != nil { + h.t.Fatalf("callActor %q: %v", h.name, err) + } + validateCounterResponse(h.t, resp, h.name, wantCount, wantCount) +} + +func (h *revertHarness) suspendSync(ctx context.Context) string { + h.t.Helper() + suspended, err := e2e.RetryAborted(h.t, ctx, h.clients.SubstrateAPI.SuspendActor, &ateapi.SuspendActorRequest{ + Actor: h.ref, + Options: &ateapi.SuspendOptions{Mode: ateapi.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + }) + if err != nil { + h.t.Fatalf("SuspendActor(SYNCHRONOUS) %q: %v", h.name, err) + } + uri := durableSnapshotURI(suspended.GetActor().GetStatus()) + if uri == "" { + h.t.Fatalf("SuspendActor(SYNCHRONOUS) %q wrote no durable snapshot", h.name) + } + return uri +} + +func (h *revertHarness) suspendAsyncAndAwaitUpload(ctx context.Context) string { + h.t.Helper() + paused, err := e2e.RetryAborted(h.t, ctx, h.clients.SubstrateAPI.SuspendActor, &ateapi.SuspendActorRequest{ + Actor: h.ref, + Options: &ateapi.SuspendOptions{Mode: ateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}, + }) + if err != nil { + h.t.Fatalf("SuspendActor(ASYNCHRONOUS) %q: %v", h.name, err) + } + waitForActorPaused(ctx, h.t, h.clients, h.name) + + local := localSnapshot(paused.GetActor().GetStatus()) + wantGen := local.GetGeneration() + wantURI := local.GetDurableSnapshot().GetObject().GetSnapshotUri() + if wantURI == "" { + h.t.Fatalf("async suspend %q recorded no durable snapshot URI on local snapshot %v", h.name, local) + } + + // e2e installs set atelet's --snapshot-upload-delay to 2s. With the 5m + // default, this wait times out. + const uploadWait = time.Minute + var lastErr error + var lastSnapshots []*ateapi.Snapshot + deadline := time.Now().Add(uploadWait) + for time.Now().Before(deadline) { + actor, err := h.clients.SubstrateAPI.GetActor(ctx, &ateapi.GetActorRequest{Actor: h.ref}) + lastErr = err + if err == nil { + lastSnapshots = actor.GetStatus().GetSnapshots() + if snap := durableSnapshot(actor.GetStatus()); snap.GetGeneration() == wantGen { + gotURI := snap.GetDurableSnapshot().GetObject().GetSnapshotUri() + if gotURI != wantURI { + h.t.Fatalf("completed durable URI for %q = %q, want %q", h.name, gotURI, wantURI) + } + return gotURI + } + } + time.Sleep(500 * time.Millisecond) + } + h.t.Fatalf("actor %q generation %d not durable after %v: last error %v, snapshots %v", h.name, wantGen, uploadWait, lastErr, lastSnapshots) + return "" +} + +func (h *revertHarness) revertAndAssert(ctx context.Context, wantURI string) *ateapi.Actor { + h.t.Helper() + reverted, err := e2e.RetryAborted(h.t, ctx, h.clients.SubstrateAPI.RevertActor, &ateapi.RevertActorRequest{Actor: h.ref}) + if err != nil { + h.t.Fatalf("RevertActor %q: %v", h.name, err) + } + actor := reverted.GetActor() + st := actor.GetStatus() + if got := st.GetState(); got != ateapi.ActorState_ACTOR_STATE_SUSPENDED { + h.t.Fatalf("state after RevertActor %q = %v, want SUSPENDED", h.name, got) + } + if got := durableSnapshotURI(st); got != wantURI { + h.t.Fatalf("durable snapshot URI after RevertActor %q = %q, want %q", h.name, got, wantURI) + } + if hasLocalSnapshot(st) { + h.t.Fatalf("local snapshot survived RevertActor %q: %v", h.name, st.GetSnapshots()) + } + if st.GetWorkerAssignment() != nil { + h.t.Fatalf("worker assignment survived RevertActor %q: %v", h.name, st.GetWorkerAssignment()) + } + if got := st.GetAssignedNode(); got != "" { + h.t.Fatalf("assigned node after RevertActor %q = %q, want empty", h.name, got) + } + if st.GetCrash() != nil { + h.t.Fatalf("crash status survived RevertActor %q: %v", h.name, st.GetCrash()) + } + return actor } From 67c6a6b344533587122fdebc0f0444d56fc17c94 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Wed, 28 Oct 2026 03:55:27 +0000 Subject: [PATCH 26/53] [bugbash] hack/teardown.sh: add --cluster-scoped to tear down one cluster (#3296) `hack/teardown.sh --all` deletes resources that every Substrate cluster in the project shares, so cleaning up one cluster broke image pulls and snapshot transfers on the others. Fixes #3227 - `--cluster-scoped` deletes only the cluster, its buckets and the bindings on them. - `--all` still deletes everything setup-gcp created, including the dashboards, the transfer service account and the project-level IAM bindings. The usage text lists those steps as project-wide. - The README says to use `--cluster-scoped` while other clusters share the project. Assisted by Claude Code. I reviewed the diff. --------- Signed-off-by: Haven Xia --- README.md | 5 ++++- hack/teardown.sh | 37 ++++++++++++++++++++++++------------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 702da9c8ba..acdcecaf84 100644 --- a/README.md +++ b/README.md @@ -230,9 +230,12 @@ go run ./cmd/ate-setup delete all If you need to delete the resources created by the setup script, you can use the provided script `hack/teardown.sh`. This script will delete resources in the reverse order of creation and handles partial failures gracefully. ```bash -./hack/teardown.sh --all +./hack/teardown.sh [--all|--cluster-scoped] ``` +* `--all` deletes everything, including the dashboards, the snapshot transfer service account and the project-level IAM bindings, which every Substrate cluster in the project shares. +* `--cluster-scoped` deletes only this cluster, its buckets and the bindings on them. Use it while other Substrate clusters still use the project. + Or run individual teardown steps as needed (see `./hack/teardown.sh` for available options). #### Tearing down local `kind` resources diff --git a/hack/teardown.sh b/hack/teardown.sh index 68360cfc15..8651b6f8d8 100755 --- a/hack/teardown.sh +++ b/hack/teardown.sh @@ -50,15 +50,17 @@ transfer_service_account() { function usage() { echo "Usage: $0 [options]" echo "Options:" - echo " --revoke-gke-node-permissions Revoke GKE nodes permission to pull images" - echo " --revoke-atelet-permissions Revoke atelet's project-level IAM bindings" echo " --delete-iam-policy-bindings Delete bucket IAM policy bindings for atelet, ate-api-server and the transfer account" - echo " --delete-transfer-service-account Delete the snapshot transfer service account" echo " --delete-snapshot-bucket Delete snapshot bucket" echo " --delete-gvisor-node-pool Delete gVisor node pool" echo " --delete-cluster Delete GKE cluster" + echo " --cluster-scoped Run the steps above (reverse order of setup)" + echo "Project-wide options. These affect every Substrate cluster in the project:" echo " --delete-dashboards Delete the Substrate monitoring dashboards" - echo " --all Run all teardown steps (reverse order of setup)" + echo " --delete-transfer-service-account Delete the snapshot transfer service account" + echo " --revoke-atelet-permissions Revoke atelet's project-level IAM bindings" + echo " --revoke-gke-node-permissions Revoke GKE nodes permission to pull images, on every cluster that uses the default compute service account" + echo " --all Run every step above (reverse order of setup)" exit 1 } @@ -189,6 +191,19 @@ delete_cluster() { --quiet || true } +# delete_buckets_and_cluster deletes the buckets, the node pool and the cluster. +delete_buckets_and_cluster() { + delete_snapshot_bucket + # Deleting the cluster removes its node pools, so this does not insist on a + # pool name a caller (e.g. an installer) may not track. + if [ -n "${NODE_POOL_NAME:-}" ]; then + delete_gvisor_node_pool + else + echo "NODE_POOL_NAME not set; skipping node pool deletion (the cluster deletion removes its pools)" + fi + delete_cluster +} + # --- Main Logic --- if [ "$#" -eq 0 ]; then usage @@ -204,21 +219,17 @@ while [[ "$#" -gt 0 ]]; do --delete-gvisor-node-pool) delete_gvisor_node_pool ;; --delete-cluster) delete_cluster ;; --delete-dashboards) delete_dashboards ;; + --cluster-scoped) + delete_iam_policy_bindings + delete_buckets_and_cluster + ;; --all) delete_dashboards delete_iam_policy_bindings delete_transfer_service_account revoke_atelet_permissions revoke_gke_node_permissions - delete_snapshot_bucket - # Deleting the cluster removes its node pools, so --all does not insist - # on a pool name a caller (e.g. an installer) may not track. - if [ -n "${NODE_POOL_NAME:-}" ]; then - delete_gvisor_node_pool - else - echo "NODE_POOL_NAME not set; skipping node pool deletion (the cluster deletion removes its pools)" - fi - delete_cluster + delete_buckets_and_cluster ;; *) usage ;; esac From f20af35fbffde81943bbd03f30661869c454928f Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Wed, 28 Oct 2026 04:53:16 +0000 Subject: [PATCH 27/53] benchmarking: what to set and watch before a dense run (#2499) A dense run meets limits the demo path never reaches. One new section lists them with the setting for each: reaching the API and router without `kubectl port-forward`, PostgreSQL slots and probe timeouts, the parking budget, synchronized bursts against the ext_proc breaker, the kernel neighbor table, restores under full load, kubelet CPU reservation, and what to watch during the run. Docs only. --------- Signed-off-by: Davanum Srinivas --- benchmarking/README.md | 67 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/benchmarking/README.md b/benchmarking/README.md index c584d1aad3..2f315fb39d 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -700,6 +700,73 @@ You must have enabled otel tracing for your cluster to view traces. You can find trace IDs by viewing the `logs` tab in the Locust UI +## Running at scale + +A dense run meets limits the demo path never reaches. The ones met so far, and +what to set before the run: + +* **Reaching the API and the router from outside the cluster.** The Locust + and boomer workers in this document run in the cluster and reach the + Services directly. A load generator outside it must not go through + `kubectl port-forward`: the forward resets under load and takes every + in-flight resume and pause down with it. Expose `svc/api` and + `svc/atenet-router` through a NodePort or LoadBalancer Service instead. + The port-forwards in this document are for the UIs only. +* **PostgreSQL connection slots.** Each ateapi replica opens up to + `pool_max_conns` connections in its read/write pool plus 5 for its owner + and watch pools. Unset, pgx sizes the pool at the node's CPU count (at + least 4), not the container's limit, so on a large node one replica can + take most of PostgreSQL's default `max_connections` of 100. Set it with + `--ateapi-postgres-pool-max-conns` on ate-setup and keep + `replicas x (pool max + 5)` under `max_connections` with slack for `psql` + and migrations (see + [connection sizing](../docs/dev/cloud-sql-scaling-guide.md#connection-sizing-for-a-target-throughput)). + To raise `max_connections`, add a `-c` argument to the StatefulSet in + `manifests/ate-install/postgres/postgres.yaml`, or install with + `--cluster-size size10`, which sets 300 and pins the pool at 64 but sizes + PostgreSQL for a dedicated node. When the slots run out, requests fail with + `INTERNAL` and `remaining connection slots are reserved`. +* **PostgreSQL probes.** Under load the probes miss their default 1 s + timeout. Three misses of the liveness probe (`pg_isready`) restart + postgres mid-run; three misses of the readiness probe (`nc`) drop the pod + from the Service, and ateapi loses its connections without a restart. Give + both `timeoutSeconds: 5` and `failureThreshold: 6`, and give the postgres + container CPU and memory requests sized for the run: on a saturated node + the request is its share of the CPU, and a starved postgres is what misses + the probes. +* **Parking budget.** `--parked-request-budget` (default `5s`) has to cover a + pause-and-resume cycle under load, tens of seconds, times the + oversubscription (live actors per worker slot); at 2x that is minutes. Watch + `atenet.router.parking.wait.duration{outcome="budget_exhausted"}`. See + [docs/request-parking.md](../docs/request-parking.md). +* **Synchronized bursts.** Clients that step in lockstep send one request per + actor at the same instant. Above the ext_proc circuit breaker Envoy answers + `503 ... reset reason: overflow` and the router never sees it. Jitter the + clients, or size `--extproc-max-requests`. +* **Kernel neighbor table.** Every sandbox has a link-local address, and + `net.ipv4.neigh.default.gc_thresh3` is global across network namespaces. + Above about a thousand sandboxes per host, raise the thresholds + (`gc_thresh1/2/3` = 4096/16384/32768). The symptom is an idle host with + `connection timed out` to its sandboxes and `arp_cache: neighbor table + overflow!` in `dmesg`. +* **Restores under full load.** A node's restore throughput falls as its + sandboxes burn CPU, and a restore wave onto a saturated node makes the + wakeup probe time out: the actor is crashed. Substrate does not steer + restores away from busy nodes, so keep CPU headroom instead: fewer resident + sandboxes per node, and admission (creates per second) paced to the restore + rate. +* **The kubelet.** Hundreds of sandboxes per node starve its heartbeat; after + the 300 s toleration the taint manager evicts every pod on the node. + Reserve CPU for it (`systemReserved`, `kubeReserved`) before going above a + few hundred sandboxes per node, and watch node `Ready`. + +Besides the Locust statistics, watch `ate.actor.crashes` (a crash is a finding, +not noise), `atenet.router.parking.*`, atelet restore and checkpoint p99, +`pg_stat_activity` against `max_connections`, node conditions, and `dmesg` for +OOM and neighbor-table lines. ateapi logs every RPC at INFO and its container +log rotates at 10 MiB, so read lifecycle events from the collector rather than +from `kubectl logs`. + ## Benchmark output files A run writes the following to `--dest`. Each run produces them fresh; none of From 6d7b3f49f6154d52a1de46c94a1481868a5ed042 Mon Sep 17 00:00:00 2001 From: Bo-Cheng Chu Date: Wed, 28 Oct 2026 06:05:42 +0000 Subject: [PATCH 28/53] Assert HTTP/2 in egress CONNECT access log and reduce /mux concurrency (#3339) `TestActorEgressMultiplexing` previously sent 250 concurrent requests in `/mux` to overflow the 100-stream per-connection limit and scraped Envoy admin metrics to check HTTP/2 multiplexing. Include `protocol=%PROTOCOL%` in the Envoy egress CONNECT access log (`manifests/ate-install/atenet-egress.yaml`), assert `protocol=HTTP/2` in `assertEgressGatewayConnect`, and reduce `muxRequestCount` in `demos/egress` from 250 to 10 concurrent requests. This verifies that the cluster negotiated HTTP/2 and multiplexed the concurrent streams without needing to overflow 100 streams or scrape Envoy admin metrics in `TestActorEgressMultiplexing`. AI use: AI-assisted, I reviewed and tested it. - [x] Tests pass - [x] Docs updated Signed-off-by: Bo-Cheng Chu --- .../internal/router/egressmanifest_test.go | 15 ++++ demos/egress/main.go | 2 +- .../e2e/suites/networking/accesslog_test.go | 51 +++++++++++++ .../e2e/suites/networking/networking_test.go | 71 +++++++------------ manifests/ate-install/atenet-egress.yaml | 2 +- 5 files changed, 93 insertions(+), 48 deletions(-) diff --git a/cmd/atenet/internal/router/egressmanifest_test.go b/cmd/atenet/internal/router/egressmanifest_test.go index 9ce84c8a09..609a9d4a3d 100644 --- a/cmd/atenet/internal/router/egressmanifest_test.go +++ b/cmd/atenet/internal/router/egressmanifest_test.go @@ -17,6 +17,7 @@ package router import ( "regexp" "slices" + "strings" "testing" "time" @@ -247,3 +248,17 @@ func TestEgressManifestsLogPathsWithoutTheQuery(t *testing.T) { t.Fatal("no access log formats found; the manifest changed shape and this test is checking nothing") } } + +func TestEgressManifestsConnectAccessLogIncludesProtocol(t *testing.T) { + t.Parallel() + logs := chainAccessLogsByLeg(bootstrapTree(t))["connect"] + if len(logs) == 0 { + t.Fatal("no connect access log found; the manifest changed shape and this test is checking nothing") + } + for i, al := range logs { + format := str(child(child(child(al, "typed_config"), "log_format"), "text_format_source"), "inline_string") + if !strings.Contains(format, "protocol=%PROTOCOL%") { + t.Errorf("connect access_log[%d] format %q does not log protocol=%%PROTOCOL%%", i, format) + } + } +} diff --git a/demos/egress/main.go b/demos/egress/main.go index 8664c6254f..4b4689bbce 100644 --- a/demos/egress/main.go +++ b/demos/egress/main.go @@ -54,7 +54,7 @@ const ( maxWebSocketMessageSize = 64 << 10 maxWebSocketHandshakeBody = 4 << 10 maxConnectAnswerBody = 4 << 10 - muxRequestCount = 250 + muxRequestCount = 10 muxBodySize = 64 // Keep connection around longer than ext_proc cache freshness (10s) postBodyDelay = 15 * time.Second diff --git a/internal/e2e/suites/networking/accesslog_test.go b/internal/e2e/suites/networking/accesslog_test.go index 21c937428b..43fd7c135b 100644 --- a/internal/e2e/suites/networking/accesslog_test.go +++ b/internal/e2e/suites/networking/accesslog_test.go @@ -141,3 +141,54 @@ func TestProtocolGatewayAccessLogMatches(t *testing.T) { t.Fatal("opaque CONNECT evidence must not satisfy the MITM matcher") } } + +func TestEgressConnectAccessLogMatches(t *testing.T) { + for _, test := range []struct { + name, container, line string + want bool + }{ + { + name: "envoy HTTP/2 CONNECT", + container: "envoy", + line: `[egress] authority=10.96.0.7:80 peer_san=spiffe://substrate-actor.local/worker-for-actor/networking-e2e/egress-mux-1 peer_serial=1 protocol=HTTP/2 code=200 flags=- up_bytes=0 down_bytes=0`, + want: true, + }, + { + name: "envoy HTTP/1.1 CONNECT rejected", + container: "envoy", + line: `[egress] authority=10.96.0.7:80 peer_san=spiffe://substrate-actor.local/worker-for-actor/networking-e2e/egress-mux-1 peer_serial=1 protocol=HTTP/1.1 code=200 flags=- up_bytes=0 down_bytes=0`, + want: false, + }, + { + name: "envoy missing protocol rejected", + container: "envoy", + line: `[egress] authority=10.96.0.7:80 peer_san=spiffe://substrate-actor.local/worker-for-actor/networking-e2e/egress-mux-1 peer_serial=1 code=200 flags=- up_bytes=0 down_bytes=0`, + want: false, + }, + { + name: "envoy wrong port", + container: "envoy", + line: `[egress] authority=10.96.0.7:443 peer_san=spiffe://substrate-actor.local/worker-for-actor/networking-e2e/egress-mux-1 peer_serial=1 protocol=HTTP/2 code=200 flags=- up_bytes=0 down_bytes=0`, + want: false, + }, + { + name: "envoy wrong actor", + container: "envoy", + line: `[egress] authority=10.96.0.7:80 peer_san=spiffe://substrate-actor.local/worker-for-actor/networking-e2e/other peer_serial=1 protocol=HTTP/2 code=200 flags=- up_bytes=0 down_bytes=0`, + want: false, + }, + { + name: "agentgateway tunnel", + container: "agentgateway", + line: `CONNECT tunnel terminated target=10.96.0.7:80 actor_name=egress-mux-1 actor_uid=uid-1 atespace=networking-e2e`, + want: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + got := egressConnectAccessLogMatches(gatewayAccessLogLine{container: test.container, text: test.line}, "networking-e2e", "egress-mux-1", "80") + if got != test.want { + t.Fatalf("egressConnectAccessLogMatches() = %t, want %t", got, test.want) + } + }) + } +} diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index d26489c09c..da3503fb38 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -141,7 +141,6 @@ func TestActorEgress(t *testing.T) { func TestActorEgressMultiplexing(t *testing.T) { ctx := context.Background() - dataplane := e2e.CurrentAtenetDataplane() origin := egressHTTPTarget() target := e2e.DeployServerPod(t, ctx, origin) @@ -154,11 +153,6 @@ func TestActorEgressMultiplexing(t *testing.T) { actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} waitForActorRoute(t, ctx, router, actorRef) - beforeScrape, err := dataplane.ScrapeMetrics(ctx) - if err != nil { - t.Fatalf("ScrapeMetrics before /mux: %v", err) - } - url := fmt.Sprintf("http://%s/fetch", target.Address()) payload, err := json.Marshal(map[string]string{"url": url}) if err != nil { @@ -173,24 +167,6 @@ func TestActorEgressMultiplexing(t *testing.T) { t.Logf("Actor egress mux of %s succeeded; body: %s", url, body) assertEgressGatewayConnect(t, ctx, since, actorAtespace, actorName, strconv.Itoa(origin.Port)) - - afterScrape, err := dataplane.ScrapeMetrics(ctx) - if err != nil { - t.Fatalf("ScrapeMetrics after /mux: %v", err) - } - - const ( - // Different dataplane may have different stream concurrency. - wantConnectionsAtLeast = 1 - wantRequests = 250 - ) - stats := dataplane.EgressConnectStats(beforeScrape, afterScrape) - if stats.Connections != -1 && stats.Connections < wantConnectionsAtLeast { - t.Errorf("egress connect connections = %d, want at least %d", stats.Connections, wantConnectionsAtLeast) - } - if stats.Requests != -1 && stats.Requests != wantRequests { - t.Errorf("egress connect requests = %d, want %d", stats.Requests, wantRequests) - } } // egressHTTPSTarget returns an origin whose /healthz answers with body over @@ -356,40 +332,43 @@ func postThroughEgressActorUntil(t *testing.T, ctx context.Context, router *e2e. } // assertEgressGatewayConnect waits for the atenet-egress log to show a CONNECT -// to port opened by actorName. Envoy logs authenticated peer details in its -// successful CONNECT access record; AgentGateway logs the terminated tunnel. +// to port opened by actorName. Envoy logs authenticated peer details and the +// negotiated HTTP/2 protocol in its successful CONNECT access record; +// AgentGateway logs the terminated tunnel. func assertEgressGatewayConnect(t *testing.T, ctx context.Context, since metav1.Time, atespace, actorName, port string) { t.Helper() want := fmt.Sprintf("a CONNECT to port %s by actor %s/%s", port, atespace, actorName) waitForAccessLog(t, ctx, since, want, func(lines []gatewayAccessLogLine) bool { for _, line := range lines { - switch line.container { - case "envoy": - authority, ok := accessLogField(line.text, "authority") - if !ok { - continue - } - if !strings.HasSuffix(authority, ":"+port) { - continue - } - spiffeSlug := "/worker-for-actor/" + atespace + "/" + actorName - if !strings.Contains(line.text, spiffeSlug) { - continue - } + if egressConnectAccessLogMatches(line, atespace, actorName, port) { + t.Logf("egress gateway tunneled the request: %s", line.text) return true - case "agentgateway": - target, ok := accessLogField(line.text, "target") - if ok && strings.HasSuffix(target, ":"+port) && - protocolGatewayAccessLogMatches(line, resources.ActorRef{Atespace: atespace, Name: actorName}, target, true) { - t.Logf("egress gateway tunneled the request: %s", line.text) - return true - } } } return false }) } +func egressConnectAccessLogMatches(line gatewayAccessLogLine, atespace, actorName, port string) bool { + switch line.container { + case "envoy": + authority, ok := accessLogField(line.text, "authority") + if !ok || !strings.HasSuffix(authority, ":"+port) { + return false + } + if protocol, ok := accessLogField(line.text, "protocol"); !ok || protocol != "HTTP/2" { + return false + } + spiffeSlug := "/worker-for-actor/" + atespace + "/" + actorName + return strings.Contains(line.text, spiffeSlug) + case "agentgateway": + target, ok := accessLogField(line.text, "target") + return ok && strings.HasSuffix(target, ":"+port) && + protocolGatewayAccessLogMatches(line, resources.ActorRef{Atespace: atespace, Name: actorName}, target, true) + } + return false +} + type gatewayAccessLogLine struct { container string text string diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index f5d2f5ddc1..231c44c9a5 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -140,7 +140,7 @@ data: "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog log_format: text_format_source: - inline_string: "[egress] authority=%REQ(:AUTHORITY)% peer_san=%DOWNSTREAM_PEER_URI_SAN% peer_serial=%DOWNSTREAM_PEER_SERIAL% code=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% up_bytes=%BYTES_RECEIVED% down_bytes=%BYTES_SENT%\n" + inline_string: "[egress] authority=%REQ(:AUTHORITY)% peer_san=%DOWNSTREAM_PEER_URI_SAN% peer_serial=%DOWNSTREAM_PEER_SERIAL% protocol=%PROTOCOL% code=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% up_bytes=%BYTES_RECEIVED% down_bytes=%BYTES_SENT%\n" route_config: name: connect_route virtual_hosts: From 8f0237461a0bd05492ef345b0e7b7ff2ed3c07f5 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Wed, 28 Oct 2026 14:52:12 +0000 Subject: [PATCH 29/53] The GCP Secret Manager credential provider moved to agent-substrate-plugins/credprovider-gcp-secret-manager (#3365) We're keeping a set of basic functionality that allows us to boot up a kind cluster and get substrate working in core, but additional plugins / implementations are moving out. This is in-line with the cloud-provider decoupling proposal. We're doing this now, during the GA push, so that we don't change the publish locations underneath users. We have not yet published any pre-built binaries, we will for the next tag cut. Following the naming agreed in #2595, the GCP Secret Manager credential provider now lives in [agent-substrate-plugins/credprovider-gcp-secret-manager](https://github.com/agent-substrate-plugins/credprovider-gcp-secret-manager), with module path `plugins.go.ate.dev/credprovider-gcp-secret-manager`. It's git history moved with it, with full attribution (direct push for initial setup, so I am not the committer, the original authors arE) Part of #3161 AI use: AI-assisted, I used Claude Code to assist with staging the new repository and prepare this change, and reviewed both. - [x] Tests pass - [x] Docs updated --------- Signed-off-by: Benjamin Elder --- .github/workflows/govulncheck.yaml | 1 - .github/workflows/pr-workflow.yaml | 16 - AGENTS.md | 2 +- docs/dev/best-practices/logging.md | 5 - docs/dev/code-layout.md | 18 +- docs/egress-credential-injection.md | 4 +- docs/metrics/substrate.yaml | 5 - internal/plugins/README.md | 19 - internal/plugins/gcp-secret-manager/.ko.yaml | 21 - internal/plugins/gcp-secret-manager/Makefile | 56 -- internal/plugins/gcp-secret-manager/README.md | 282 --------- .../cmd/gsm-credential-provider/clientpool.go | 79 --- .../clientpool_test.go | 111 ---- .../gsm-credential-provider/imports_test.go | 94 --- .../cmd/gsm-credential-provider/main.go | 239 ------- .../cmd/gsm-credential-provider/main_test.go | 219 ------- .../gcp-secret-manager/config/deployment.yaml | 106 ---- .../config/kustomization.yaml | 23 - .../config/project-policy.yaml | 35 -- .../gcp-secret-manager/config/service.yaml | 30 - .../config/serviceaccount.yaml | 25 - internal/plugins/gcp-secret-manager/go.mod | 47 -- internal/plugins/gcp-secret-manager/go.sum | 94 --- .../internal/credbundle/credbundle.go | 186 ------ .../internal/credbundle/credbundle_test.go | 429 ------------- .../gcp-secret-manager/internal/mtls/mtls.go | 98 --- .../internal/mtls/mtls_test.go | 400 ------------ .../internal/provider/projectauthz.go | 121 ---- .../internal/provider/projectauthz_test.go | 143 ----- .../internal/provider/provider.go | 328 ---------- .../internal/provider/provider_test.go | 586 ------------------ 31 files changed, 18 insertions(+), 3804 deletions(-) delete mode 100644 internal/plugins/README.md delete mode 100644 internal/plugins/gcp-secret-manager/.ko.yaml delete mode 100644 internal/plugins/gcp-secret-manager/Makefile delete mode 100644 internal/plugins/gcp-secret-manager/README.md delete mode 100644 internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/clientpool.go delete mode 100644 internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/clientpool_test.go delete mode 100644 internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/imports_test.go delete mode 100644 internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main.go delete mode 100644 internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main_test.go delete mode 100644 internal/plugins/gcp-secret-manager/config/deployment.yaml delete mode 100644 internal/plugins/gcp-secret-manager/config/kustomization.yaml delete mode 100644 internal/plugins/gcp-secret-manager/config/project-policy.yaml delete mode 100644 internal/plugins/gcp-secret-manager/config/service.yaml delete mode 100644 internal/plugins/gcp-secret-manager/config/serviceaccount.yaml delete mode 100644 internal/plugins/gcp-secret-manager/go.mod delete mode 100644 internal/plugins/gcp-secret-manager/go.sum delete mode 100644 internal/plugins/gcp-secret-manager/internal/credbundle/credbundle.go delete mode 100644 internal/plugins/gcp-secret-manager/internal/credbundle/credbundle_test.go delete mode 100644 internal/plugins/gcp-secret-manager/internal/mtls/mtls.go delete mode 100644 internal/plugins/gcp-secret-manager/internal/mtls/mtls_test.go delete mode 100644 internal/plugins/gcp-secret-manager/internal/provider/projectauthz.go delete mode 100644 internal/plugins/gcp-secret-manager/internal/provider/projectauthz_test.go delete mode 100644 internal/plugins/gcp-secret-manager/internal/provider/provider.go delete mode 100644 internal/plugins/gcp-secret-manager/internal/provider/provider_test.go diff --git a/.github/workflows/govulncheck.yaml b/.github/workflows/govulncheck.yaml index 35255104c9..0b1bbdd528 100644 --- a/.github/workflows/govulncheck.yaml +++ b/.github/workflows/govulncheck.yaml @@ -32,7 +32,6 @@ jobs: # The root module, then each module kept apart from it. work-dir: - . - - internal/plugins/gcp-secret-manager steps: - id: govulncheck uses: golang/govulncheck-action@032d45514ae346b1db93c04b0c90b841c370344f # v1.1.0 diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 00940a6e29..a6a0b009c1 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -85,22 +85,6 @@ jobs: --jsonfile "${ARTIFACTS}/junittool.json" \ --format pkgname-and-test-fails --format-hide-empty-pkg \ -- -race ./... - # Kept apart from the root module so it can move to its own repository. - - name: gcp-secret-manager credential provider tests - working-directory: internal/plugins/gcp-secret-manager - run: | - go -C ../../../tools/junittool run . register "${ARTIFACTS}/gcp-secret-manager.xml" - ../../../hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/gcp-secret-manager.xml" \ - --jsonfile "${ARTIFACTS}/gcp-secret-manager.json" \ - --format pkgname-and-test-fails --format-hide-empty-pkg \ - -- -race ./... - # The root verifiers stop at the module's go.mod, so it gets its own run: - # gofmt, go vet and the tidy check, then golangci-lint with the root config. - - name: gcp-secret-manager credential provider verify - working-directory: internal/plugins/gcp-secret-manager - run: | - make verify - ../../../hack/run-tool.sh golangci-lint run --config ../../../.golangci.yaml ./... - name: Cache cargo uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: diff --git a/AGENTS.md b/AGENTS.md index 269a52b78f..c393aafa88 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,7 +95,7 @@ tools/ # Standalone Go tools (go run ./tools/) for Dev/CI | Internal proto (atelet / ateworker) | `internal/proto/` | | Dev/CI scripts | `hack/` | | Standalone Go dev/CI tools | `tools/` with its own `go.mod` | -| Self-contained plugin that nothing in the repo depends on | `internal/plugins/` with its own `go.mod`, importing only `api/` and `pkg/`; see `internal/plugins/README.md` | +| Plugin for a specific backend, implementing a plugin API like `api/credprovider` | Its own repository in the `agent-substrate-plugins` org; see "Plugins" in `docs/dev/code-layout.md` | See `docs/dev/code-layout.md` for the full rationale and per-directory details. diff --git a/docs/dev/best-practices/logging.md b/docs/dev/best-practices/logging.md index a1c441a4d7..13cbda5f9c 100644 --- a/docs/dev/best-practices/logging.md +++ b/docs/dev/best-practices/logging.md @@ -24,11 +24,6 @@ builds its own handler gets no masking. These processes build their own today: - `podcertcontroller` - `atenet sdsmint` - `boomer-worker` -- plugins under `internal/plugins/`, such as `gsm-credential-provider`. A - plugin there imports only `pkg/` (see its README), so it cannot call - `serverboot`. If `gsm-credential-provider` logged the - `FetchSecretResponse` it serves, `opaque_bytes` would show in clear. The - plugins under `cmd/` use `serverboot` and are masked. The handler masks labeled fields in a proto it is handed as an attribute value (`slog.Any("resp", resp)`), inside a `slog.Group`, as the result of a diff --git a/docs/dev/code-layout.md b/docs/dev/code-layout.md index 088e83fa4d..94c7b7dfbe 100644 --- a/docs/dev/code-layout.md +++ b/docs/dev/code-layout.md @@ -33,9 +33,7 @@ a commitment to backwards-compatibility and discoverability. **`internal/`** is for Go packages that are shared across multiple binaries within this module but are not part of any external API. The Go toolchain enforces that nothing outside `go.ate.dev/substrate` can import -these. The exception is [`internal/plugins/`](../../internal/plugins/README.md): -self-contained plugins, each a Go module of its own, in a temporary location -that nothing else in the repo should depend on or import. +these. > **Use `pkg/` sparingly.** Once the project reaches GA, any exported type, > function, or field in `pkg/` becomes subject to a compatibility guarantee — @@ -62,6 +60,20 @@ Each `.proto` sits next to its generated Go. Proto packages are convention instead: `package v1`, imported as `atev1`. A new version of an API goes in a new directory next to the old one. +### Plugins + +Core keeps a reference implementation of each plugin API, such as the +Kubernetes Secrets credential provider. We use these to develop and test +substrate on `kind`. + +Anyone can implement a plugin API and run their own plugin. We also host +additional implementations in the +[agent-substrate-plugins](https://github.com/agent-substrate-plugins) +organization, subject to approval by the substrate maintainers. These import +only substrate's public `api/` and `pkg/` packages, and each repository is named +for the plugin API it implements and then its backend, as in +[credprovider-gcp-secret-manager](https://github.com/agent-substrate-plugins/credprovider-gcp-secret-manager). + ### `cmd//` Each subdirectory of `cmd/` corresponds to one compiled binary: diff --git a/docs/egress-credential-injection.md b/docs/egress-credential-injection.md index daa1ea9adf..4891b452c8 100644 --- a/docs/egress-credential-injection.md +++ b/docs/egress-credential-injection.md @@ -114,7 +114,7 @@ interpret. A URI the provider refuses denies the request with 403. [The reference provider](#the-reference-provider). * **Google Cloud Secret Manager** (provider class `secretmanager.googleapis.com`) — see its - [README](../internal/plugins/gcp-secret-manager/README.md#credential-uris). + [README](https://github.com/agent-substrate-plugins/credprovider-gcp-secret-manager#credential-uris). ### Actor JWTs @@ -319,7 +319,7 @@ wrong provider. The gateway reads only the URI host, to confirm the URI targets the provider it serves. Everything after the host is the provider's to interpret. -`internal/plugins/gcp-secret-manager` is such a provider for Google Cloud Secret Manager. +[credprovider-gcp-secret-manager](https://github.com/agent-substrate-plugins/credprovider-gcp-secret-manager) is such a provider for Google Cloud Secret Manager. **Trust model.** The gateway dials the provider over mTLS with its own pod identity, `spiffe://cluster.local/ns/ate-system/sa/atenet-egress`, and every diff --git a/docs/metrics/substrate.yaml b/docs/metrics/substrate.yaml index f5e1aba961..b110828441 100644 --- a/docs/metrics/substrate.yaml +++ b/docs/metrics/substrate.yaml @@ -237,11 +237,6 @@ blind_spots: flag, and a 1% sample of the rest. The runtime keys access_log.min_status and access_log.sample on the admin port restore full logging while you debug. - - area: GCP Secret Manager credential provider - code: internal/plugins/gcp-secret-manager - brief: > - This plugin has no metrics. It does not install the otelgrpc server - handler. Thus rpc.server.call.duration has no credprovider series for it. - area: ate-idp-server code: cmd/ate-idp-server brief: > diff --git a/internal/plugins/README.md b/internal/plugins/README.md deleted file mode 100644 index 22bc10ebc1..0000000000 --- a/internal/plugins/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Plugins - -Each directory here is a self-contained plugin: a Go module of its own that -plugs into substrate through its public APIs. - -This is a temporary location. Each plugin will move to a repository of its -own, and its path here goes away when it does. Until then: - -- **Do not depend on these paths.** Nothing else in the repository should - build on, script against or link into a plugin's directory. The only - references from outside a plugin are its own CI steps and pointers from the - docs. -- **Do not import these packages.** Code outside a plugin must not import it. - Go already refuses most such imports, since a plugin's packages sit under its - own `internal/` or are `main` packages. -- **A plugin imports only substrate's public `api/` and `pkg/` packages**, so - it can move out as it is. Its `go.mod` replaces the substrate module with this - checkout, so it builds against the current tree. - diff --git a/internal/plugins/gcp-secret-manager/.ko.yaml b/internal/plugins/gcp-secret-manager/.ko.yaml deleted file mode 100644 index 9d60cbabcb..0000000000 --- a/internal/plugins/gcp-secret-manager/.ko.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Substrate's base image, pinned by multi-platform index digest. Set -# KO_DEFAULTPLATFORMS to override the platforms, e.g. host-only for kind. -defaultBaseImage: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78 - -defaultPlatforms: - - linux/amd64 - - linux/arm64 diff --git a/internal/plugins/gcp-secret-manager/Makefile b/internal/plugins/gcp-secret-manager/Makefile deleted file mode 100644 index b769489cc9..0000000000 --- a/internal/plugins/gcp-secret-manager/Makefile +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ko pushes to KO_DOCKER_REPO, e.g. localhost:5001 for kind. --base-import-paths -# names the image gsm-credential-provider, without a hash suffix. -KO ?= go run github.com/google/ko@v0.19.1 -KO_FLAGS ?= --base-import-paths - -# The toolchain's own gofmt, not whichever one is first on PATH. -GOFMT := $(shell go env GOROOT)/bin/gofmt - -.PHONY: all -all: verify test - -.PHONY: build -build: - go build ./... - -.PHONY: test -test: - go test -race ./... - -.PHONY: fmt -fmt: - $(GOFMT) -w . - -.PHONY: verify -verify: - @out="$$($(GOFMT) -l .)"; if [ -n "$$out" ]; then echo "gofmt needed on:"; echo "$$out"; exit 1; fi - go vet ./... - go mod tidy -diff - -# Builds the image and applies the manifests to the current kubectl context. -.PHONY: deploy -deploy: - kubectl kustomize config | $(KO) apply $(KO_FLAGS) -f - - -# Renders the manifests with the image pinned by digest, for a release. -.PHONY: release-manifest -release-manifest: - kubectl kustomize config | $(KO) resolve $(KO_FLAGS) -f - - -.PHONY: undeploy -undeploy: - kubectl delete -k config --ignore-not-found diff --git a/internal/plugins/gcp-secret-manager/README.md b/internal/plugins/gcp-secret-manager/README.md deleted file mode 100644 index f58695f378..0000000000 --- a/internal/plugins/gcp-secret-manager/README.md +++ /dev/null @@ -1,282 +0,0 @@ -# gcp-secret-manager credential provider - -> [!NOTE] -> This provider is hosted in the substrate repository temporarily. It will move -> to a dedicated repository in the near future. - -A [substrate](https://github.com/agent-substrate/substrate) credential provider -backed by Google Cloud Secret Manager. It implements substrate's -`CredentialProvider` gRPC API (`api/credprovider/v1`) for egress -credential injection: given an `ate-secret://secretmanager.googleapis.com/...` -URI, it reads the secret version from Secret Manager and returns its payload, -so the secret stays in Secret Manager and substrate never stores it. - -This directory is a Go module of its own, so that it can move out as it is. - -## Endpoint - -The manifests in `config/` deploy the provider into substrate's `ate-system` -namespace: - -| | | -|---|---| -| Provider name | `secretmanager.googleapis.com`, the host of its `ate-secret://` credential URIs | -| Address | `gsm-credential-provider.ate-system.svc:50051` | -| API | `substrate.credprovider.v1.CredentialProvider/FetchSecret` over mutual TLS | - -The provider's serving certificate names the Service's DNS name, so callers -must use exactly this address. - -## Credential URIs - -The URI path is the resource name of a secret version, optionally followed by a -key into the version's JSON payload. A regional secret names its location after -the project, as its resource name does: - -``` -ate-secret://secretmanager.googleapis.com/projects//secrets//versions/[/keys/] -ate-secret://secretmanager.googleapis.com/projects//locations//secrets//versions/[/keys/] -``` - -- `` is a project ID or number. -- `` is a region, such as `us-central1`. The provider reads a - regional secret from its region's endpoint, - `secretmanager..rep.googleapis.com`. The URI host stays - `secretmanager.googleapis.com`: it names the provider, not the endpoint. -- `` is 1-255 letters, digits, underscores or dashes. -- `` is required: a positive integer or `latest`. A URI without a - version is refused rather than read as `latest`. -- `` names a top-level key of a payload holding a JSON object, such as - `{"api_key": "..."}`. It is a single path segment other than `.` or `..`, so - a key containing `/` or a character that needs percent-encoding cannot be - named. - -User information, queries, fragments, percent-encoding and empty path segments -(a doubled or trailing slash) are refused. - -## Responses - -Without a key, `FetchSecret` returns the payload verbatim in `opaque_bytes`. -With a key, it returns the key's value, which must be a JSON string, with its -JSON escapes decoded. When Secret Manager supplies a CRC32C checksum, the -provider verifies the payload against it first. - -The egress gateway trims trailing CR and LF from the value and refuses an empty -value or one containing any other control character, since the value becomes -part of an HTTP header. A multi-line credential, such as a PEM-encoded key, -cannot be injected. - -Each Secret Manager read may take up to `--fetch-timeout`, 3s by default. The -gateway sets no deadline of its own and Envoy abandons the request after its 5s -ext_proc message timeout, so keep the flag below that. - -The gateway answers `Unavailable` and `DeadlineExceeded` with a retryable 503 -and every other code with a 403: - -| Condition | Code | -|---|---| -| The URI is malformed, lacks a version, or names another provider | `InvalidArgument` | -| Secret Manager rejects the resource name | `InvalidArgument` | -| The actor's atespace is not granted the URI's project, or the actor identity is unusable | `PermissionDenied` | -| The provider's Google identity may not access the secret, or its credentials are refused | `PermissionDenied` | -| The secret or version does not exist or has no payload, or the payload's JSON object has no such key | `NotFound` | -| The version is disabled or destroyed, including a `latest` whose newest version is disabled | `FailedPrecondition` | -| With a key, the payload is not a JSON object or the key's value is not a string | `FailedPrecondition` | -| Secret Manager does not answer in time | `DeadlineExceeded` | -| The payload fails its CRC32C check, or any other failure, such as an outage or exhausted quota | `Unavailable` | - -Error messages name the key but never quote the payload. - -## Security model - -- **Only the egress gateway may call.** The provider serves mutual TLS with a - certificate from substrate's servicedns signer, and admits a caller only if - its certificate chains to substrate's podidentity trust bundle and carries - the egress gateway's SPIFFE ID. `--injector-identity` names that ID (the - gateway is what injects credentials); `config/` sets it to the gateway in - `ate-system`. -- **Each atespace resolves secrets only in the projects its policy grants.** - The provider takes the atespace from the actor SPIFFE ID the gateway sends - and, before calling Secret Manager, refuses with `PermissionDenied` a URI - whose project the `--project-policy-file` policy does not list for it. The - policy is default-deny, like the Kubernetes Secrets provider's namespace - policy. A project is matched as the URI writes it, so list - it by the ID or number your URIs use. -- **The provider's Google identity bounds the rest.** Within a granted project, - an atespace reaches whatever that identity can read, so grant it access only - to the secrets some atespace needs. - -## Install - -Prerequisites: - -- A GKE cluster with Workload Identity Federation, running substrate v0.2.0 or - later. The provider gets its Google credentials from GKE's metadata server, - and its serving certificate and its callers' trust bundle from substrate's - pod certificate controller. -- A registry for `KO_DOCKER_REPO`. The Makefile runs [ko](https://ko.build) - with `go run`. - -**1. Grant Secret Manager access.** The provider reads secrets through Workload -Identity Federation for GKE, as its Kubernetes ServiceAccount -`gsm-credential-provider` in `ate-system`. Grant it access to each secret it -serves in one of two ways. `` is the project that holds the GKE -cluster, which need not be the secret's ``. - -*Grant the Kubernetes ServiceAccount directly.* Nothing else needs configuring: - -```bash -gcloud secrets add-iam-policy-binding --project \ - --role roles/secretmanager.secretAccessor \ - --member "principal://iam.googleapis.com/projects//locations/global/workloadIdentityPools/.svc.id.goog/subject/ns/ate-system/sa/gsm-credential-provider" -``` - -*Or act as a Google service account*, for example one that already holds the -access. Grant the Google service account access to the secret, and let the -Kubernetes ServiceAccount impersonate it: - -```bash -GSA=gsm-credential-provider@.iam.gserviceaccount.com -gcloud iam service-accounts create gsm-credential-provider --project -gcloud secrets add-iam-policy-binding --project \ - --role roles/secretmanager.secretAccessor --member "serviceAccount:$GSA" -gcloud iam service-accounts add-iam-policy-binding "$GSA" --project \ - --role roles/iam.workloadIdentityUser \ - --member "serviceAccount:.svc.id.goog[ate-system/gsm-credential-provider]" -``` - -Then uncomment the `iam.gke.io/gcp-service-account` annotation in -`config/serviceaccount.yaml`, set to that email, before deploying the provider. - -For a regional secret, see step 5. - -**2. Apply the project policy.** Edit -[`config/project-policy.yaml`](config/project-policy.yaml), the -`gsm-credential-provider-project-policy` ConfigMap, to list the projects each -atespace may resolve secrets in, then apply it from this directory. The -provider's pod mounts it, so it must exist first. `make deploy` leaves it out, -so a redeploy never overwrites it. - -```bash -kubectl apply -f config/project-policy.yaml -``` - -The provider reads the policy once, at startup, and exits if it is malformed. -After editing it, restart the provider: - -```bash -kubectl -n ate-system rollout restart deployment/gsm-credential-provider -``` - -**3. Deploy the provider.** From this directory: - -```bash -KO_DOCKER_REPO= make deploy -kubectl -n ate-system rollout status deployment/gsm-credential-provider -``` - -The provider exits at startup, logging `could not find default credentials`, -if it finds no Application Default Credentials. Off GKE, on a kind cluster say, -no metadata server supplies them; mount a service account key into the pod and -point `GOOGLE_APPLICATION_CREDENTIALS` at it. A missing IAM grant does not stop -the provider: it fails each fetch with `PermissionDenied`. - -**4. Point the egress gateway at the provider.** From the repository root: - -```bash -hack/install-ate.sh --deploy-atenet \ - --credential-provider='{"name":"secretmanager.googleapis.com","address":"gsm-credential-provider.ate-system.svc:50051"}' -``` - -This sets the `--credential-provider-*` flags on the `ext-proc` container of -the `atenet-egress` Deployment and restarts it. It also redeploys the atenet -router, building its images from the checkout with ko; for a published release, -put `ATE_IMAGE_REPO= ATE_IMAGE_TAG=` in front of the command. The -gateway serves one provider at a time, so this replaces any previous one. To -confirm: - -```bash -kubectl -n ate-system get deployment atenet-egress -o yaml | grep credential-provider -``` - -**5. Store a secret.** The payload is the raw value: - -```bash -printf '%s' "$TOKEN" | gcloud secrets create example-api-token --project --data-file=- -``` - -Its URI is -`ate-secret://secretmanager.googleapis.com/projects//secrets/example-api-token/versions/1`, -or `.../versions/latest` to follow new versions. - -To keep several credentials in one secret, store a JSON object and name the key: - -```bash -jq -cn --arg v "$TOKEN" '{api_key: $v}' | gcloud secrets create example-api --project --data-file=- -``` - -`.../secrets/example-api/versions/1/keys/api_key` then resolves to `$TOKEN`. - -For a regional secret, add `--location ` to the `gcloud secrets` -commands, here and in step 1, and point gcloud at the regional endpoint: -`--location` alone still sends the request to the global endpoint, which fails. - -```bash -printf '%s' "$TOKEN" | \ - CLOUDSDK_API_ENDPOINT_OVERRIDES_SECRETMANAGER="https://secretmanager..rep.googleapis.com/" \ - gcloud secrets create example-api-token --project --location --data-file=- -``` - -Its URI names the location: -`ate-secret://secretmanager.googleapis.com/projects//locations//secrets/example-api-token/versions/1`. -A project's policy grant covers its regional secrets too. - -**Uninstall.** Redeploy the gateway without the provider first: removing the -provider while the gateway still points at it makes every credential fetch -fail. - -```bash -hack/install-ate.sh --deploy-atenet --credential-provider='{"enabled":false}' # from the repository root -make undeploy # from this directory -``` - -`make undeploy` removes the provider's ServiceAccount, Deployment and Service, -and leaves the project policy in place. - -## Flags - -| Flag | Default | Purpose | -|---|---|---| -| `--listen-address` | `:50051` | gRPC listen address | -| `--health-address` | `:9090` | HTTP address for `/healthz` and `/readyz` | -| `--server-cred-bundle` | required | Serving credential bundle: PKCS#8 key and certificate chain | -| `--client-ca-file` | required | Trust bundle the caller's certificate must chain to | -| `--injector-identity` | required | SPIFFE ID of substrate's egress gateway, the only caller the provider accepts; `spiffe://cluster.local/ns/ate-system/sa/atenet-egress` in a default install | -| `--project-policy-file` | required | Atespace→project policy YAML | -| `--fetch-timeout` | `3s` | Time limit for one Secret Manager read; keep it under the gateway's 5s ext_proc message timeout | -| `--log-level` | `info` | `debug`, `info`, `warn` or `error` | -| `--drain-grace` | `5s` | How long in-flight calls may finish on shutdown | - -## Development - -```bash -make test # go test -race ./... -make verify # gofmt, go vet, go mod tidy check -make fmt # gofmt -w . -make release-manifest # manifests with the image pinned by digest -``` - -`go.mod` replaces the substrate module with the enclosing checkout, so the -provider builds and tests against the current `credprovider` API. - -The module imports nothing of substrate's but its public `api/` and `pkg/` -packages, so it can be copied out as it is; -`TestImportsOnlySubstratePublicPackages` fails on any other substrate import. -For that reason, two packages are copies kept in step by hand: -`internal/credbundle` of substrate's `internal/credbundle`, and `internal/mtls` -of the server credentials in `cmd/credential-provider/kubernetes-secrets`. - -To move the module to its own repository, rename the module path in `go.mod`, -in the Go imports, and in the `ko://` image of `config/deployment.yaml`, then -drop the `replace` directive so the `require` of a released substrate version -applies. diff --git a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/clientpool.go b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/clientpool.go deleted file mode 100644 index efc383bf1c..0000000000 --- a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/clientpool.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "sync" - - "go.ate.dev/substrate/internal/plugins/gcp-secret-manager/internal/provider" -) - -// secretManagerClient is a closable SecretAccessor. -type secretManagerClient interface { - provider.SecretAccessor - Close() error -} - -// clientPool holds the global Secret Manager client plus one per region, -// created on first use: a regional secret is served only by its region's -// endpoint. -type clientPool struct { - global secretManagerClient - // newRegional must not use a request's context: a client keeps the context - // it was created with, e.g. to refresh credentials. - newRegional func(location string) (secretManagerClient, error) - - mu sync.Mutex - regional map[string]secretManagerClient -} - -// For returns the client for location ("" for global). A failed creation is -// not cached, so the next call retries. -func (p *clientPool) For(location string) (provider.SecretAccessor, error) { - if location == "" { - return p.global, nil - } - p.mu.Lock() - defer p.mu.Unlock() - if c, ok := p.regional[location]; ok { - return c, nil - } - c, err := p.newRegional(location) - if err != nil { - return nil, err - } - if p.regional == nil { - p.regional = make(map[string]secretManagerClient) - } - p.regional[location] = c - return c, nil -} - -// Close closes every client the pool holds. -func (p *clientPool) Close() error { - p.mu.Lock() - defer p.mu.Unlock() - errs := []error{p.global.Close()} - for _, c := range p.regional { - errs = append(errs, c.Close()) - } - return errors.Join(errs...) -} - -// regionalEndpoint returns the endpoint serving a region's secrets. -func regionalEndpoint(location string) string { - return "secretmanager." + location + ".rep.googleapis.com:443" -} diff --git a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/clientpool_test.go b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/clientpool_test.go deleted file mode 100644 index 37576545ae..0000000000 --- a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/clientpool_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "errors" - "testing" - - secretmanagerpb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" - "github.com/googleapis/gax-go/v2" -) - -// fakeClient is a Secret Manager client that records whether it was closed. -type fakeClient struct { - name string - closed bool -} - -func (f *fakeClient) AccessSecretVersion(context.Context, *secretmanagerpb.AccessSecretVersionRequest, ...gax.CallOption) (*secretmanagerpb.AccessSecretVersionResponse, error) { - return nil, errors.New("not used") -} - -func (f *fakeClient) Close() error { - f.closed = true - return nil -} - -// The global client serves ""; each region's client is made once, on first -// use; a failed creation is retried; Close closes every client. -func TestClientPool(t *testing.T) { - global := &fakeClient{name: "global"} - made := map[string]int{} - var failNext bool - var regional []*fakeClient - p := &clientPool{ - global: global, - newRegional: func(location string) (secretManagerClient, error) { - made[location]++ - if failNext { - failNext = false - return nil, errors.New("cannot create client") - } - c := &fakeClient{name: location} - regional = append(regional, c) - return c, nil - }, - } - - for _, tc := range []struct { - location string - want string - }{ - {"", "global"}, - {"us-central1", "us-central1"}, - {"us-central1", "us-central1"}, - {"europe-west4", "europe-west4"}, - } { - c, err := p.For(tc.location) - if err != nil { - t.Fatalf("For(%q) error = %v", tc.location, err) - } - if got := c.(*fakeClient).name; got != tc.want { - t.Errorf("For(%q) = client %q, want %q", tc.location, got, tc.want) - } - } - if made["us-central1"] != 1 || made["europe-west4"] != 1 || made[""] != 0 { - t.Errorf("clients made per location = %v, want one per region and none for global", made) - } - - failNext = true - if _, err := p.For("asia-east1"); err == nil { - t.Fatal("For(asia-east1) succeeded while client creation fails, want an error") - } - if _, err := p.For("asia-east1"); err != nil { - t.Fatalf("For(asia-east1) after a failed attempt: %v, want a fresh client", err) - } - if made["asia-east1"] != 2 { - t.Errorf("clients made for asia-east1 = %d, want 2: a failure is not remembered", made["asia-east1"]) - } - - if err := p.Close(); err != nil { - t.Fatalf("Close() error = %v", err) - } - if !global.closed { - t.Error("Close() left the global client open") - } - for _, c := range regional { - if !c.closed { - t.Errorf("Close() left the %s client open", c.name) - } - } -} - -func TestRegionalEndpoint(t *testing.T) { - if got, want := regionalEndpoint("us-central1"), "secretmanager.us-central1.rep.googleapis.com:443"; got != want { - t.Errorf("regionalEndpoint(us-central1) = %q, want %q", got, want) - } -} diff --git a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/imports_test.go b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/imports_test.go deleted file mode 100644 index 6703be0ce1..0000000000 --- a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/imports_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bufio" - "go/parser" - "go/token" - "os" - "path/filepath" - "strconv" - "strings" - "testing" -) - -// substrateModule is the module this one is hosted in until it moves out. -const substrateModule = "go.ate.dev/substrate" - -// This module moves to a repository of its own by being copied, so it may -// import only substrate's public packages. Go's internal rule goes by import -// path, and this module's path sits under substrate's, so an import of -// substrate's internal/ packages would compile here and break only after the -// move. -func TestImportsOnlySubstratePublicPackages(t *testing.T) { - root := filepath.Join("..", "..") - self := modulePath(t, filepath.Join(root, "go.mod")) - - fset := token.NewFileSet() - err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - // The go command ignores these, and so does the rule. - if name := d.Name(); path != root && (strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") || name == "testdata") { - return filepath.SkipDir - } - return nil - } - if !strings.HasSuffix(path, ".go") { - return nil - } - f, err := parser.ParseFile(fset, path, nil, parser.ImportsOnly) - if err != nil { - return err - } - for _, spec := range f.Imports { - imp, err := strconv.Unquote(spec.Path.Value) - if err != nil { - return err - } - switch { - case imp == self || strings.HasPrefix(imp, self+"/"): - case strings.HasPrefix(imp, substrateModule+"/api/"), strings.HasPrefix(imp, substrateModule+"/pkg/"): - case imp == substrateModule || strings.HasPrefix(imp, substrateModule+"/"): - t.Errorf("%s imports %s; this module may import only %s/api/... and %s/pkg/...", path, imp, substrateModule, substrateModule) - } - } - return nil - }) - if err != nil { - t.Fatalf("walking the module: %v", err) - } -} - -// modulePath reads the module path from a go.mod file. -func modulePath(t *testing.T, goMod string) string { - t.Helper() - f, err := os.Open(goMod) - if err != nil { - t.Fatalf("opening %s: %v", goMod, err) - } - defer f.Close() - scanner := bufio.NewScanner(f) - for scanner.Scan() { - if path, ok := strings.CutPrefix(strings.TrimSpace(scanner.Text()), "module "); ok { - return strings.TrimSpace(path) - } - } - t.Fatalf("%s names no module", goMod) - return "" -} diff --git a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main.go b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main.go deleted file mode 100644 index 94d46f8227..0000000000 --- a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main.go +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Command gsm-credential-provider serves substrate's CredentialProvider gRPC -// API from Google Cloud Secret Manager. The egress gateway calls it to resolve -// ate-secret:// URIs for credential injection; it is the only component in that -// path with Secret Manager access. -package main - -import ( - "context" - "errors" - "flag" - "fmt" - "log/slog" - "net" - "net/http" - "os" - "os/signal" - "runtime/debug" - "sync/atomic" - "syscall" - "time" - - secretmanager "cloud.google.com/go/secretmanager/apiv1" - "google.golang.org/api/option" - "google.golang.org/grpc" - - "go.ate.dev/substrate/api/credprovider/v1" - - "go.ate.dev/substrate/internal/plugins/gcp-secret-manager/internal/mtls" - "go.ate.dev/substrate/internal/plugins/gcp-secret-manager/internal/provider" -) - -var ( - listenAddr = flag.String("listen-address", ":50051", "gRPC listen address") - healthAddr = flag.String("health-address", ":9090", "HTTP listen address for /healthz and /readyz") - serverBundle = flag.String("server-cred-bundle", "", "credential bundle (PKCS#8 key and certificate chain) presented for serving TLS (required)") - clientCAFile = flag.String("client-ca-file", "", "trust bundle the caller's client certificate must chain to (required)") - // No default: the gateway's namespace and ServiceAccount are known only to - // the deployment. - injectorIdentity = flag.String("injector-identity", "", "SPIFFE ID the caller's client certificate must carry: substrate's egress gateway, e.g. spiffe:///ns//sa/atenet-egress (required)") - projectPolicyFile = flag.String("project-policy-file", "", "path to the atespace→project authorization YAML (required)") - logLevel = flag.String("log-level", "info", "one of debug, info, warn, error") - drainGrace = flag.Duration("drain-grace", 5*time.Second, "how long to wait for in-flight RPCs on shutdown before a hard stop") - // Stays under the egress gateway's 5s ext_proc message timeout, after which - // Envoy abandons the request. - fetchTimeout = flag.Duration("fetch-timeout", 3*time.Second, "how long a single Secret Manager read may take; keep it under the egress gateway's ext_proc message timeout") -) - -// version is set at build time with -ldflags "-X main.version=v0.1.0". -var version string - -func main() { - flag.Parse() - - var level slog.Level - if err := level.UnmarshalText([]byte(*logLevel)); err != nil { - fmt.Fprintf(os.Stderr, "invalid --log-level %q: %v\n", *logLevel, err) - os.Exit(2) - } - slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: level}))) - - slog.Info("starting gsm-credential-provider", slog.String("version", buildVersion())) - if err := run(context.Background()); err != nil { - slog.Error("gsm-credential-provider exited with error", slog.Any("err", err)) - os.Exit(1) - } -} - -func run(ctx context.Context) error { - if *serverBundle == "" { - return errors.New("--server-cred-bundle is required") - } - if *clientCAFile == "" { - return errors.New("--client-ca-file is required") - } - if *injectorIdentity == "" { - return errors.New("--injector-identity is required") - } - if *projectPolicyFile == "" { - return errors.New("--project-policy-file is required") - } - if *fetchTimeout <= 0 { - return fmt.Errorf("--fetch-timeout must be positive, got %v", *fetchTimeout) - } - authz, err := provider.LoadProjectAuthorizer(*projectPolicyFile) - if err != nil { - return fmt.Errorf("project policy: %w", err) - } - slog.Info("loaded project authorization policy", slog.String("file", *projectPolicyFile)) - - creds, err := mtls.ServerCredentials(mtls.Config{ - ServerBundle: *serverBundle, - ClientCAFile: *clientCAFile, - CallerIdentity: *injectorIdentity, - }) - if err != nil { - return fmt.Errorf("server credentials: %w", err) - } - slog.Info("admitting callers", slog.String("client_ca", *clientCAFile), slog.String("required_san", *injectorIdentity)) - - // Application Default Credentials (Workload Identity on GKE). NewClient fails - // without any, so the pod exits rather than serving; a missing IAM grant - // surfaces per FetchSecret as PermissionDenied. - smClient, err := secretmanager.NewClient(ctx) - if err != nil { - return fmt.Errorf("secret manager client: %w", err) - } - // Regional clients use ctx, which outlives every request, because a client - // keeps the context it was created with. - clients := &clientPool{ - global: smClient, - newRegional: func(location string) (secretManagerClient, error) { - return secretmanager.NewClient(ctx, option.WithEndpoint(regionalEndpoint(location))) - }, - } - defer clients.Close() - - srv := grpc.NewServer(grpc.Creds(creds)) - credprovider.RegisterCredentialProviderServer(srv, provider.NewServer(clients.For, authz, *fetchTimeout)) - - grpcLis, err := (&net.ListenConfig{}).Listen(ctx, "tcp", *listenAddr) - if err != nil { - return fmt.Errorf("listen on %s: %w", *listenAddr, err) - } - healthLis, err := (&net.ListenConfig{}).Listen(ctx, "tcp", *healthAddr) - if err != nil { - return fmt.Errorf("listen on %s: %w", *healthAddr, err) - } - var ready atomic.Bool - healthSrv := &http.Server{Handler: healthHandler(&ready), ReadHeaderTimeout: 10 * time.Second} - healthDone := make(chan error, 1) - go func() { healthDone <- healthSrv.Serve(healthLis) }() - - shutdownCtx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) - defer stop() - shutdownErr := make(chan error, 1) - go func() { - var err error - select { - case <-shutdownCtx.Done(): - slog.Info("shutting down") - case err = <-healthDone: - err = fmt.Errorf("health server: %w", err) - } - ready.Store(false) - gracefulStop(srv, *drainGrace) - _ = healthSrv.Close() - shutdownErr <- err - }() - - ready.Store(true) - slog.Info("gsm-credential-provider listening", slog.String("address", grpcLis.Addr().String()), slog.String("health_address", healthLis.Addr().String())) - // ErrServerStopped means shutdown began before Serve started: not a failure. - if err := srv.Serve(grpcLis); err != nil && !errors.Is(err, grpc.ErrServerStopped) { - return fmt.Errorf("serving: %w", err) - } - return <-shutdownErr -} - -// gracefulStop lets in-flight RPCs finish for up to grace, then forces the -// server down. -func gracefulStop(srv *grpc.Server, grace time.Duration) { - done := make(chan struct{}) - go func() { - srv.GracefulStop() - close(done) - }() - select { - case <-done: - case <-time.After(grace): - slog.Warn("graceful shutdown timed out; forcing stop", slog.Duration("grace", grace)) - srv.Stop() - } -} - -// healthHandler serves /healthz, which always succeeds so a draining pod is not -// restarted, and /readyz, which succeeds only while ready is set. -func healthHandler(ready *atomic.Bool) http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte("ok")) - }) - mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { - if !ready.Load() { - http.Error(w, "not ready", http.StatusServiceUnavailable) - return - } - _, _ = w.Write([]byte("ok")) - }) - return mux -} - -// buildVersion returns the release or module version, else "dev", plus the VCS -// revision when the build recorded one. -func buildVersion() string { - v := version - info, ok := debug.ReadBuildInfo() - if !ok { - if v == "" { - return "dev" - } - return v - } - if v == "" && info.Main.Version != "" && info.Main.Version != "(devel)" { - v = info.Main.Version - } - if v == "" { - v = "dev" - } - var revision, modified string - for _, s := range info.Settings { - switch s.Key { - case "vcs.revision": - revision = s.Value - case "vcs.modified": - modified = s.Value - } - } - if revision == "" { - return v - } - if modified == "true" { - revision += "-dirty" - } - return v + " commit=" + revision -} diff --git a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main_test.go b/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main_test.go deleted file mode 100644 index 206d0f9c3c..0000000000 --- a/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider/main_test.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "errors" - "io/fs" - "net" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "sync/atomic" - "testing" - "time" - - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/test/bufconn" - - "go.ate.dev/substrate/api/credprovider/v1" -) - -// /healthz passes regardless of readiness; /readyz follows the ready flag. -func TestHealthHandler(t *testing.T) { - var ready atomic.Bool - h := healthHandler(&ready) - - get := func(path string) int { - rec := httptest.NewRecorder() - h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) - return rec.Code - } - - if got := get("/healthz"); got != http.StatusOK { - t.Errorf("/healthz before ready = %d, want 200", got) - } - if got := get("/readyz"); got != http.StatusServiceUnavailable { - t.Errorf("/readyz before ready = %d, want 503", got) - } - - ready.Store(true) - if got := get("/readyz"); got != http.StatusOK { - t.Errorf("/readyz when ready = %d, want 200", got) - } - - ready.Store(false) - if got := get("/readyz"); got != http.StatusServiceUnavailable { - t.Errorf("/readyz while draining = %d, want 503", got) - } - if got := get("/healthz"); got != http.StatusOK { - t.Errorf("/healthz while draining = %d, want 200", got) - } -} - -// run rejects a missing flag, an unreadable file or a non-positive fetch -// timeout before it dials anything or binds a port. -func TestRunRequiresTLSFlags(t *testing.T) { - const caller = "spiffe://cluster.local/ns/ate-system/sa/atenet-egress" - policy := filepath.Join(t.TempDir(), "project-policy.yaml") - if err := os.WriteFile(policy, []byte("policies: []\n"), 0o600); err != nil { - t.Fatalf("writing the policy: %v", err) - } - absent := filepath.Join(t.TempDir(), "absent") - tests := []struct { - name string - serverBundle string - clientCAFile string - injectorIdentity string - policyFile string - fetchTimeout time.Duration - wantErr string - // wantIs, if set, must be in the error chain; wantErr alone matches any - // failure of the same step. - wantIs error - }{ - {name: "no server bundle", clientCAFile: "/ca.pem", injectorIdentity: caller, policyFile: policy, wantErr: "--server-cred-bundle"}, - {name: "no client CA", serverBundle: "/bundle.pem", injectorIdentity: caller, policyFile: policy, wantErr: "--client-ca-file"}, - {name: "no injector identity", serverBundle: "/bundle.pem", clientCAFile: "/ca.pem", policyFile: policy, wantErr: "--injector-identity"}, - {name: "no project policy", serverBundle: "/bundle.pem", clientCAFile: "/ca.pem", injectorIdentity: caller, wantErr: "--project-policy-file"}, - {name: "project policy unreadable", serverBundle: "/bundle.pem", clientCAFile: "/ca.pem", injectorIdentity: caller, policyFile: absent, wantErr: "project policy", wantIs: fs.ErrNotExist}, - {name: "fetch timeout not positive", serverBundle: "/bundle.pem", clientCAFile: "/ca.pem", injectorIdentity: caller, policyFile: policy, fetchTimeout: -time.Second, wantErr: "--fetch-timeout"}, - {name: "client CA unreadable", serverBundle: "/bundle.pem", clientCAFile: absent, injectorIdentity: caller, policyFile: policy, wantErr: "server credentials", wantIs: fs.ErrNotExist}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - origBundle, origCA, origIdentity, origPolicy, origTimeout := *serverBundle, *clientCAFile, *injectorIdentity, *projectPolicyFile, *fetchTimeout - t.Cleanup(func() { - *serverBundle, *clientCAFile, *injectorIdentity, *projectPolicyFile, *fetchTimeout = origBundle, origCA, origIdentity, origPolicy, origTimeout - }) - *serverBundle, *clientCAFile, *injectorIdentity, *projectPolicyFile = tc.serverBundle, tc.clientCAFile, tc.injectorIdentity, tc.policyFile - if tc.fetchTimeout != 0 { - *fetchTimeout = tc.fetchTimeout - } - - err := run(context.Background()) - if err == nil || !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("run() error = %v, want one mentioning %q", err, tc.wantErr) - } - if tc.wantIs != nil && !errors.Is(err, tc.wantIs) { - t.Errorf("run() error = %v, want one wrapping %v", err, tc.wantIs) - } - }) - } -} - -func TestBuildVersionPrefersExplicitVersion(t *testing.T) { - orig := version - t.Cleanup(func() { version = orig }) - - version = "v9.9.9" - if got := buildVersion(); !strings.HasPrefix(got, "v9.9.9") { - t.Errorf("buildVersion() = %q, want it to start with v9.9.9", got) - } - version = "" - if got := buildVersion(); got == "" { - t.Error("buildVersion() is empty without an explicit version") - } -} - -// blockingProvider blocks FetchSecret until release is closed or the call is -// canceled, closing entered once the call starts. -type blockingProvider struct { - credprovider.UnimplementedCredentialProviderServer - entered chan struct{} - release chan struct{} -} - -func (p *blockingProvider) FetchSecret(ctx context.Context, _ *credprovider.FetchSecretRequest) (*credprovider.FetchSecretResponse, error) { - close(p.entered) - select { - case <-p.release: - return &credprovider.FetchSecretResponse{}, nil - case <-ctx.Done(): - return nil, ctx.Err() - } -} - -// startBlockingServer serves p in memory and starts one FetchSecret against it, -// returning the server and the call's result. -func startBlockingServer(t *testing.T, p *blockingProvider) (*grpc.Server, <-chan error) { - t.Helper() - lis := bufconn.Listen(1 << 20) - srv := grpc.NewServer() - credprovider.RegisterCredentialProviderServer(srv, p) - go func() { _ = srv.Serve(lis) }() - t.Cleanup(srv.Stop) - - conn, err := grpc.NewClient("passthrough:///bufconn", - grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) }), - grpc.WithTransportCredentials(insecure.NewCredentials())) - if err != nil { - t.Fatalf("grpc.NewClient: %v", err) - } - t.Cleanup(func() { _ = conn.Close() }) - - result := make(chan error, 1) - go func() { - _, err := credprovider.NewCredentialProviderClient(conn).FetchSecret(context.Background(), &credprovider.FetchSecretRequest{}) - result <- err - }() - select { - case <-p.entered: - case <-time.After(10 * time.Second): - t.Fatal("FetchSecret never reached the server") - } - return srv, result -} - -// gracefulStop returns once an in-flight call completes within the grace period. -func TestGracefulStopDrains(t *testing.T) { - p := &blockingProvider{entered: make(chan struct{}), release: make(chan struct{})} - srv, result := startBlockingServer(t, p) - - stopped := make(chan struct{}) - go func() { - gracefulStop(srv, time.Minute) - close(stopped) - }() - close(p.release) - select { - case <-stopped: - case <-time.After(10 * time.Second): - t.Fatal("gracefulStop did not return after the in-flight call finished") - } - if err := <-result; err != nil { - t.Errorf("in-flight FetchSecret = %v, want it to complete", err) - } -} - -// gracefulStop cuts off a call still running when the grace period ends. -func TestGracefulStopForcesAfterGrace(t *testing.T) { - const grace = 100 * time.Millisecond - p := &blockingProvider{entered: make(chan struct{}), release: make(chan struct{})} - srv, result := startBlockingServer(t, p) - - start := time.Now() - gracefulStop(srv, grace) - if elapsed := time.Since(start); elapsed < grace || elapsed > 50*grace { - t.Errorf("gracefulStop returned after %v, want about %v", elapsed, grace) - } - if err := <-result; err == nil { - t.Error("in-flight FetchSecret succeeded, want it cut off") - } -} diff --git a/internal/plugins/gcp-secret-manager/config/deployment.yaml b/internal/plugins/gcp-secret-manager/config/deployment.yaml deleted file mode 100644 index 66838d737a..0000000000 --- a/internal/plugins/gcp-secret-manager/config/deployment.yaml +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: gsm-credential-provider - namespace: ate-system - labels: - app: gsm-credential-provider -spec: - replicas: 1 - selector: - matchLabels: - app: gsm-credential-provider - template: - metadata: - labels: - app: gsm-credential-provider - spec: - serviceAccountName: gsm-credential-provider - securityContext: - runAsUser: 65532 - runAsGroup: 65532 - runAsNonRoot: true - containers: - - name: gsm-credential-provider - image: ko://go.ate.dev/substrate/internal/plugins/gcp-secret-manager/cmd/gsm-credential-provider - args: - - "--listen-address=:50051" - - "--health-address=:9090" - # A servicedns certificate for gsm-credential-provider.ate-system.svc, - # the name the egress gateway pins when it dials the provider. - - "--server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - # Admit only the egress gateway (atenet-egress in ate-system). Change - # the namespace if substrate runs elsewhere. - - "--client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem" - - "--injector-identity=spiffe://cluster.local/ns/ate-system/sa/atenet-egress" - # Enforce the atespace→project authorization policy (default-deny). - - "--project-policy-file=/etc/gsm-credential-provider/project-policy.yaml" - - "--log-level=info" - ports: - - name: grpc - containerPort: 50051 - - name: health - containerPort: 9090 - readinessProbe: - httpGet: - path: /readyz - port: health - periodSeconds: 10 - livenessProbe: - httpGet: - path: /healthz - port: health - periodSeconds: 20 - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - volumeMounts: - - name: project-policy - mountPath: /etc/gsm-credential-provider - readOnly: true - - name: servicedns - mountPath: /run/servicedns.podcert.ate.dev - readOnly: true - - name: podidentity - mountPath: /run/podidentity.podcert.ate.dev - readOnly: true - volumes: - # Created by project-policy.yaml, which the kustomization omits so a - # redeploy never overwrites an edited policy. - - name: project-policy - configMap: - name: gsm-credential-provider-project-policy - # The serving certificate, issued for the Services that select this pod. - - name: servicedns - projected: - sources: - - podCertificate: - signerName: servicedns.podcert.ate.dev/identity - keyType: ECDSAP256 - credentialBundlePath: credential-bundle.pem - # The CA the egress gateway's client certificate chains to. - - name: podidentity - projected: - sources: - - clusterTrustBundle: - signerName: podidentity.podcert.ate.dev/identity - labelSelector: - matchLabels: - podcert.ate.dev/canarying: live - path: trust-bundle.pem diff --git a/internal/plugins/gcp-secret-manager/config/kustomization.yaml b/internal/plugins/gcp-secret-manager/config/kustomization.yaml deleted file mode 100644 index b3c7419fab..0000000000 --- a/internal/plugins/gcp-secret-manager/config/kustomization.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Deploys into substrate's ate-system namespace, which this base neither creates -# nor deletes. The image is a ko:// reference: apply with `make deploy`, or -# render a digest-pinned manifest with `make release-manifest`. -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- serviceaccount.yaml -- deployment.yaml -- service.yaml diff --git a/internal/plugins/gcp-secret-manager/config/project-policy.yaml b/internal/plugins/gcp-secret-manager/config/project-policy.yaml deleted file mode 100644 index 3532b4cd62..0000000000 --- a/internal/plugins/gcp-secret-manager/config/project-policy.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Which Google Cloud projects each atespace may read secrets from. An actor can -# resolve a secret only if its URI's project is listed for the actor's atespace; -# an atespace that isn't listed resolves nothing. Write each project the way -# your URIs do, by ID or by number. -# -# Apply this before deploying the provider; the kustomization leaves it out so a -# redeploy doesn't overwrite it. The provider reads it only at startup, so -# restart the Deployment after editing it. -apiVersion: v1 -kind: ConfigMap -metadata: - name: gsm-credential-provider-project-policy - namespace: ate-system -data: - project-policy.yaml: | - # Actors in atespace "team-a" may use secrets in project "my-project", e.g. - # ate-secret://secretmanager.googleapis.com/projects/my-project/secrets/example-api-token/versions/1. - policies: - - atespace: team-a - allowedProjects: - - my-project diff --git a/internal/plugins/gcp-secret-manager/config/service.yaml b/internal/plugins/gcp-secret-manager/config/service.yaml deleted file mode 100644 index 743acd4a21..0000000000 --- a/internal/plugins/gcp-secret-manager/config/service.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Required for the pod to start: substrate issues the serving certificate only -# for pods a Service selects. -apiVersion: v1 -kind: Service -metadata: - name: gsm-credential-provider - namespace: ate-system -spec: - type: ClusterIP - selector: - app: gsm-credential-provider - ports: - - name: grpc - port: 50051 - targetPort: grpc - protocol: TCP diff --git a/internal/plugins/gcp-secret-manager/config/serviceaccount.yaml b/internal/plugins/gcp-secret-manager/config/serviceaccount.yaml deleted file mode 100644 index f80bd2790e..0000000000 --- a/internal/plugins/gcp-secret-manager/config/serviceaccount.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The provider's Secret Manager identity. It needs no Kubernetes RBAC. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: gsm-credential-provider - namespace: ate-system - # By default the provider reads secrets as this ServiceAccount's own Workload - # Identity principal. To act as a Google service account instead, set its - # email here (README, install step 1): - # annotations: - # iam.gke.io/gcp-service-account: gsm-credential-provider@CLUSTER_PROJECT.iam.gserviceaccount.com diff --git a/internal/plugins/gcp-secret-manager/go.mod b/internal/plugins/gcp-secret-manager/go.mod deleted file mode 100644 index b1dc19e179..0000000000 --- a/internal/plugins/gcp-secret-manager/go.mod +++ /dev/null @@ -1,47 +0,0 @@ -module go.ate.dev/substrate/internal/plugins/gcp-secret-manager - -go 1.27.0 - -toolchain go1.27.3 - -require ( - cloud.google.com/go/secretmanager v1.16.0 - github.com/googleapis/gax-go/v2 v2.21.0 - go.ate.dev/substrate v0.2.0 - google.golang.org/api v0.274.0 - google.golang.org/grpc v1.83.2 - sigs.k8s.io/yaml v1.6.0 -) - -require ( - cloud.google.com/go/auth v0.19.0 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect - cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.9.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/felixge/httpsnoop v1.1.0 // indirect - github.com/go-logr/logr v1.4.4 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect - go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect - go.opentelemetry.io/otel v1.45.0 // indirect - go.opentelemetry.io/otel/metric v1.45.0 // indirect - go.opentelemetry.io/otel/trace v1.45.0 // indirect - go.yaml.in/yaml/v2 v2.4.4 // indirect - golang.org/x/crypto v0.57.0 // indirect - golang.org/x/net v0.60.0 // indirect - golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.23.0 // indirect - golang.org/x/sys v0.48.0 // indirect - golang.org/x/text v0.42.0 // indirect - golang.org/x/time v0.15.0 // indirect - google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect - google.golang.org/protobuf v1.36.12 // indirect -) - -replace go.ate.dev/substrate => ../../.. diff --git a/internal/plugins/gcp-secret-manager/go.sum b/internal/plugins/gcp-secret-manager/go.sum deleted file mode 100644 index 179110c738..0000000000 --- a/internal/plugins/gcp-secret-manager/go.sum +++ /dev/null @@ -1,94 +0,0 @@ -cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= -cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/auth v0.19.0 h1:DGYwtbcsGsT1ywuxsIoWi1u/vlks0moIblQHgSDgQkQ= -cloud.google.com/go/auth v0.19.0/go.mod h1:2Aph7BT2KnaSFOM0JDPyiYgNh6PL9vGMiP8CUIXZ+IY= -cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= -cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= -cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.9.0 h1:89wyjxT6DL4b5rk/Nk8eBC9DHqf+JiMstrn5IEYxFw4= -cloud.google.com/go/iam v1.9.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= -cloud.google.com/go/secretmanager v1.16.0 h1:19QT7ZsLJ8FSP1k+4esQvuCD7npMJml6hYzilxVyT+k= -cloud.google.com/go/secretmanager v1.16.0/go.mod h1://C/e4I8D26SDTz1f3TQcddhcmiC3rMEl0S1Cakvs3Q= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= -github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= -github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= -github.com/envoyproxy/go-control-plane/envoy v1.37.1-0.20260812071801-353463cc7248 h1:+HVeI0tXxznym8nR2TFJ1S0rxbzER4aevVcMA/Bl/i0= -github.com/envoyproxy/go-control-plane/envoy v1.37.1-0.20260812071801-353463cc7248/go.mod h1:tjY7cZvIhlQAKRpob4GSOkZYAqtoTPtgWZXGhOTWbhQ= -github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= -github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= -github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= -github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= -github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= -github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8= -github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= -github.com/googleapis/gax-go/v2 v2.21.0 h1:h45NjjzEO3faG9Lg/cFrBh2PgegVVgzqKzuZl/wMbiI= -github.com/googleapis/gax-go/v2 v2.21.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= -github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= -github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= -go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= -go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= -go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= -go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= -go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= -go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= -go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= -go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= -go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= -go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= -go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= -go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= -go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= -golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= -golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= -golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= -golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= -golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= -golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= -golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= -gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA= -google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= -google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= -google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= -google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= -google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= -google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= -sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/plugins/gcp-secret-manager/internal/credbundle/credbundle.go b/internal/plugins/gcp-secret-manager/internal/credbundle/credbundle.go deleted file mode 100644 index acce95df2e..0000000000 --- a/internal/plugins/gcp-secret-manager/internal/credbundle/credbundle.go +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package credbundle loads Kubernetes Pod Certificate credential bundles and PEM -// trust bundles. A credential bundle is one PEM file holding a PRIVATE KEY block -// and the CERTIFICATE chain, leaf first, with or without the root. -// -// It is the server-side subset of substrate's internal/credbundle, copied so -// this module depends only on substrate's public packages. -package credbundle - -import ( - "crypto/tls" - "crypto/x509" - "encoding/pem" - "fmt" - "os" - "sync" -) - -// Loader returns a tls.Config GetCertificate function serving the credential -// bundle at path. The parse is cached and redone only when the file changes, so -// a rotation is picked up on the next handshake. -func Loader(path string) func(*tls.ClientHelloInfo) (*tls.Certificate, error) { - c := &certCache{path: path} - return func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { - return c.get() - } -} - -// PoolLoader returns a function yielding the trust pool in the PEM file at -// path, cached like Loader's. Call it per connection, e.g. from -// GetConfigForClient: a tls.Config's ClientCAs is fixed once in use, so a pool -// built at startup would miss a CA rotation. -func PoolLoader(path string) func() (*x509.CertPool, error) { - c := &poolCache{path: path} - return c.get -} - -// certCache caches a credential bundle's parse with the file stat it came from. -type certCache struct { - path string - - mu sync.Mutex - // fi is the stat taken just before cert was parsed; cert is served while a - // fresh stat matches it. - fi os.FileInfo - cert *tls.Certificate -} - -// get returns the cached bundle, re-parsing when the file's identity -// (os.SameFile), mtime or size has changed. The kubelet rotates projected -// volumes by swapping a symlink, which changes identity; mtime and size catch -// in-place rewrites. -// -// A change between the stat and the read stores newer content under the older -// stat, so the next call re-reads: the cache lags a rotation by at most one -// handshake. Errors are returned, never masked by the cache, and later calls -// retry. -func (c *certCache) get() (*tls.Certificate, error) { - c.mu.Lock() - defer c.mu.Unlock() - - fi, err := os.Stat(c.path) - if err != nil { - return nil, fmt.Errorf("while getting file info for credential bundle %q: %w", c.path, err) - } - if c.cert != nil && os.SameFile(c.fi, fi) && fi.ModTime().Equal(c.fi.ModTime()) && fi.Size() == c.fi.Size() { - return c.cert, nil - } - - cert, err := Parse(c.path) - if err != nil { - return nil, err - } - c.fi, c.cert = fi, cert - return cert, nil -} - -// poolCache is certCache for a trust bundle. -type poolCache struct { - path string - - mu sync.Mutex - fi os.FileInfo - pool *x509.CertPool -} - -// get returns the cached trust pool, re-parsing as certCache.get does. -func (c *poolCache) get() (*x509.CertPool, error) { - c.mu.Lock() - defer c.mu.Unlock() - - fi, err := os.Stat(c.path) - if err != nil { - return nil, fmt.Errorf("while getting file info for trust bundle %q: %w", c.path, err) - } - if c.pool != nil && os.SameFile(c.fi, fi) && fi.ModTime().Equal(c.fi.ModTime()) && fi.Size() == c.fi.Size() { - return c.pool, nil - } - - pool, err := ParsePool(c.path) - if err != nil { - return nil, err - } - c.fi, c.pool = fi, pool - return pool, nil -} - -// Parse reads the PKCS#8 private key and certificate chain from a credential -// bundle file. -func Parse(bundlePath string) (*tls.Certificate, error) { - bundleBytes, err := os.ReadFile(bundlePath) - if err != nil { - return nil, fmt.Errorf("while reading credential bundle: %w", err) - } - - var leafKeyBytes []byte - var chainBytes [][]byte - - for { - var block *pem.Block - block, bundleBytes = pem.Decode(bundleBytes) - if block == nil { - break - } - - switch block.Type { - case "CERTIFICATE": - chainBytes = append(chainBytes, block.Bytes) - case "PRIVATE KEY": - leafKeyBytes = block.Bytes - default: - return nil, fmt.Errorf("unknown PEM block type %q", block.Type) - } - } - - if leafKeyBytes == nil { - return nil, fmt.Errorf("no PRIVATE KEY block found") - } - - if len(chainBytes) == 0 { - return nil, fmt.Errorf("no CERTIFICATE blocks found") - } - - leafKey, err := x509.ParsePKCS8PrivateKey(leafKeyBytes) - if err != nil { - return nil, fmt.Errorf("while parsing private key: %w", err) - } - - leafCert, err := x509.ParseCertificate(chainBytes[0]) - if err != nil { - return nil, fmt.Errorf("while parsing leaf certificate: %w", err) - } - - return &tls.Certificate{ - Certificate: chainBytes, - Leaf: leafCert, - PrivateKey: leafKey, - }, nil -} - -// ParsePool reads a PEM trust bundle into a pool. A file with no certificates -// is an error, since an empty pool would silently reject every peer. -func ParsePool(path string) (*x509.CertPool, error) { - pemBytes, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("while reading trust bundle: %w", err) - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(pemBytes) { - return nil, fmt.Errorf("trust bundle %q contains no certificates", path) - } - return pool, nil -} diff --git a/internal/plugins/gcp-secret-manager/internal/credbundle/credbundle_test.go b/internal/plugins/gcp-secret-manager/internal/credbundle/credbundle_test.go deleted file mode 100644 index 16c3ebb2b7..0000000000 --- a/internal/plugins/gcp-secret-manager/internal/credbundle/credbundle_test.go +++ /dev/null @@ -1,429 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package credbundle - -import ( - "bytes" - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "errors" - "fmt" - "math/big" - "os" - "path/filepath" - "sync" - "syscall" - "testing" - "time" -) - -func TestParsePKCS8PrivateKeyBlock(t *testing.T) { - key := generateRSAKey(t) - keyDER, err := x509.MarshalPKCS8PrivateKey(key) - if err != nil { - t.Fatalf("marshal PKCS8 key: %v", err) - } - certDER := generateCertificate(t, 1) - bundle := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})...) - - bundlePath := writeBundle(t, bundle) - cert, err := Parse(bundlePath) - if err != nil { - t.Fatalf("Parse() error = %v", err) - } - if len(cert.Certificate) != 1 { - t.Fatalf("Parse() certificate chain length = %d, want 1", len(cert.Certificate)) - } - if cert.PrivateKey == nil { - t.Fatalf("Parse() private key is nil") - } - if cert.Leaf == nil { - t.Fatalf("Parse() leaf certificate is nil") - } -} - -func TestParseRejectsNonPKCS8PrivateKeyBlock(t *testing.T) { - certDER := generateCertificate(t, 1) - bundle := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(generateRSAKey(t))})...) - - bundlePath := writeBundle(t, bundle) - if _, err := Parse(bundlePath); err == nil { - t.Fatalf("Parse() error = nil, want unsupported private key block error") - } -} - -func TestLoaderServesCachedParseWhileFileUnchanged(t *testing.T) { - bundle := makeBundle(t, 7) - path := writeBundle(t, bundle) - getCert := Loader(path) - - if _, err := getCert(nil); err != nil { - t.Fatalf("Loader() first call error = %v", err) - } - - fi, err := os.Stat(path) - if err != nil { - t.Fatalf("stat bundle: %v", err) - } - // Same-length garbage with the old mtime leaves the stat unchanged; a - // re-read would fail on the garbage. - if err := os.WriteFile(path, bytes.Repeat([]byte("x"), len(bundle)), 0o600); err != nil { - t.Fatalf("overwrite bundle: %v", err) - } - if err := os.Chtimes(path, fi.ModTime(), fi.ModTime()); err != nil { - t.Fatalf("restore mtime: %v", err) - } - - cert, err := getCert(nil) - if err != nil { - t.Fatalf("Loader() with unchanged stat error = %v", err) - } - if got := leafSerial(t, cert); got != 7 { - t.Fatalf("Loader() leaf serial = %d, want cached 7", got) - } -} - -func TestLoaderPicksUpProjectedVolumeRotation(t *testing.T) { - path := writeProjectedBundle(t, makeBundle(t, 1)) - getCert := Loader(path) - - cert, err := getCert(nil) - if err != nil { - t.Fatalf("Loader() first call error = %v", err) - } - if got := leafSerial(t, cert); got != 1 { - t.Fatalf("Loader() leaf serial = %d, want 1", got) - } - - if err := rotateProjectedBundle(path, makeBundle(t, 2)); err != nil { - t.Fatalf("rotate bundle: %v", err) - } - - cert, err = getCert(nil) - if err != nil { - t.Fatalf("Loader() after rotation error = %v", err) - } - if got := leafSerial(t, cert); got != 2 { - t.Fatalf("Loader() leaf serial after rotation = %d, want 2", got) - } -} - -func TestLoaderPicksUpInPlaceRewrite(t *testing.T) { - path := writeBundle(t, makeBundle(t, 1)) - getCert := Loader(path) - - if _, err := getCert(nil); err != nil { - t.Fatalf("Loader() first call error = %v", err) - } - - fi, err := os.Stat(path) - if err != nil { - t.Fatalf("stat bundle: %v", err) - } - // Rewrite in place (same inode), bumping the mtime past coarse timestamps. - if err := os.WriteFile(path, makeBundle(t, 2), 0o600); err != nil { - t.Fatalf("rewrite bundle: %v", err) - } - bumped := fi.ModTime().Add(time.Second) - if err := os.Chtimes(path, bumped, bumped); err != nil { - t.Fatalf("bump mtime: %v", err) - } - - cert, err := getCert(nil) - if err != nil { - t.Fatalf("Loader() after rewrite error = %v", err) - } - if got := leafSerial(t, cert); got != 2 { - t.Fatalf("Loader() leaf serial after rewrite = %d, want 2", got) - } -} - -func TestLoaderErrorWhenBundleMissing(t *testing.T) { - getCert := Loader(t.TempDir() + "/absent.pem") - if _, err := getCert(nil); err == nil { - t.Fatalf("Loader() error = nil, want missing-file error") - } -} - -func TestLoaderRecoversAfterError(t *testing.T) { - bundle := makeBundle(t, 3) - path := writeBundle(t, bundle) - getCert := Loader(path) - - if _, err := getCert(nil); err != nil { - t.Fatalf("Loader() first call error = %v", err) - } - if err := os.Remove(path); err != nil { - t.Fatalf("remove bundle: %v", err) - } - if _, err := getCert(nil); err == nil { - t.Fatalf("Loader() error = nil after bundle removed, want error") - } - if err := os.WriteFile(path, bundle, 0o600); err != nil { - t.Fatalf("restore bundle: %v", err) - } - - cert, err := getCert(nil) - if err != nil { - t.Fatalf("Loader() after restore error = %v", err) - } - if got := leafSerial(t, cert); got != 3 { - t.Fatalf("Loader() leaf serial after restore = %d, want 3", got) - } -} - -func TestPoolLoaderServesCachedParseWhileFileUnchanged(t *testing.T) { - trust := makeTrustBundle(t, 5) - path := writeBundle(t, trust) - getPool := PoolLoader(path) - - first, err := getPool() - if err != nil { - t.Fatalf("PoolLoader() first call error = %v", err) - } - - fi, err := os.Stat(path) - if err != nil { - t.Fatalf("stat bundle: %v", err) - } - // Same-length garbage with the old mtime leaves the stat unchanged; a - // re-read would fail on the garbage. - if err := os.WriteFile(path, bytes.Repeat([]byte("x"), len(trust)), 0o600); err != nil { - t.Fatalf("overwrite bundle: %v", err) - } - if err := os.Chtimes(path, fi.ModTime(), fi.ModTime()); err != nil { - t.Fatalf("restore mtime: %v", err) - } - - second, err := getPool() - if err != nil { - t.Fatalf("PoolLoader() with unchanged stat error = %v", err) - } - if !second.Equal(first) { - t.Fatalf("PoolLoader() returned a re-parsed pool, want the cached one") - } -} - -func TestPoolLoaderPicksUpProjectedVolumeRotation(t *testing.T) { - path := writeProjectedBundle(t, makeTrustBundle(t, 1)) - getPool := PoolLoader(path) - - before, err := getPool() - if err != nil { - t.Fatalf("PoolLoader() first call error = %v", err) - } - - rotated := makeTrustBundle(t, 2) - if err := rotateProjectedBundle(path, rotated); err != nil { - t.Fatalf("rotate bundle: %v", err) - } - - after, err := getPool() - if err != nil { - t.Fatalf("PoolLoader() after rotation error = %v", err) - } - if after.Equal(before) { - t.Fatalf("PoolLoader() did not pick up the rotated trust bundle") - } - want, err := ParsePool(path) - if err != nil { - t.Fatalf("ParsePool() error = %v", err) - } - if !after.Equal(want) { - t.Fatalf("PoolLoader() pool does not match the rotated trust bundle") - } -} - -func TestPoolLoaderErrorWhenBundleMissing(t *testing.T) { - getPool := PoolLoader(t.TempDir() + "/absent.pem") - if _, err := getPool(); err == nil { - t.Fatalf("PoolLoader() error = nil, want missing-file error") - } -} - -func TestParsePoolRejectsBundleWithoutCertificates(t *testing.T) { - path := writeBundle(t, []byte("not a certificate")) - if _, err := ParsePool(path); err == nil { - t.Fatalf("ParsePool() error = nil, want no-certificates error") - } -} - -func TestLoaderConcurrentHandshakes(t *testing.T) { - bundles := [][]byte{makeBundle(t, 1), makeBundle(t, 2)} - path := writeProjectedBundle(t, bundles[0]) - getCert := Loader(path) - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - for i := 0; i < 25; i++ { - if err := rotateProjectedBundle(path, bundles[i%2]); err != nil { - t.Errorf("rotate bundle: %v", err) - return - } - } - }() - for g := 0; g < 4; g++ { - wg.Add(1) - go func() { - defer wg.Done() - for i := 0; i < 100; i++ { - cert, err := getCert(nil) - if err != nil { - // On macOS, stat through a symlink being renamed over - // can fail with a transient EINVAL. Linux always - // resolves the old or the new target. - if errors.Is(err, syscall.EINVAL) { - continue - } - t.Errorf("Loader() error = %v", err) - return - } - if cert.Leaf == nil { - t.Errorf("Loader() returned certificate with nil leaf") - return - } - if s := cert.Leaf.SerialNumber.Int64(); s != 1 && s != 2 { - t.Errorf("Loader() leaf serial = %d, want 1 or 2", s) - return - } - } - }() - } - wg.Wait() -} - -func generateRSAKey(t *testing.T) *rsa.PrivateKey { - t.Helper() - key, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - t.Fatalf("generate RSA key: %v", err) - } - return key -} - -func generateCertificate(t *testing.T, serial int64) []byte { - t.Helper() - template := &x509.Certificate{ - SerialNumber: big.NewInt(serial), - Subject: pkix.Name{CommonName: "api.ate-system.svc"}, - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(time.Hour), - DNSNames: []string{"api.ate-system.svc"}, - } - key := generateRSAKey(t) - der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) - if err != nil { - t.Fatalf("create certificate: %v", err) - } - return der -} - -func writeBundle(t *testing.T, bundle []byte) string { - t.Helper() - path := t.TempDir() + "/bundle.pem" - if err := os.WriteFile(path, bundle, 0o600); err != nil { - t.Fatalf("write bundle: %v", err) - } - return path -} - -// makeBundle returns a credential bundle whose leaf carries serial, so tests -// can tell bundles apart. -func makeBundle(t *testing.T, serial int64) []byte { - t.Helper() - keyDER, err := x509.MarshalPKCS8PrivateKey(generateRSAKey(t)) - if err != nil { - t.Fatalf("marshal PKCS8 key: %v", err) - } - return append( - pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: generateCertificate(t, serial)}), - pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})..., - ) -} - -// makeTrustBundle returns a one-certificate trust bundle carrying serial. -func makeTrustBundle(t *testing.T, serial int64) []byte { - t.Helper() - return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: generateCertificate(t, serial)}) -} - -func leafSerial(t *testing.T, cert *tls.Certificate) int64 { - t.Helper() - if cert == nil || cert.Leaf == nil { - t.Fatalf("parsed bundle has no leaf certificate") - } - return cert.Leaf.SerialNumber.Int64() -} - -// writeProjectedBundle lays bundle out as the kubelet's atomic writer -// (k8s.io/kubernetes/pkg/volume/util/atomic_writer.go) does and returns the -// visible /bundle.pem path: -// -// /bundle.pem -> ..data/bundle.pem -// /..data -> ..payload-/ (kubelet uses a timestamped name) -// /..payload-/bundle.pem -func writeProjectedBundle(t *testing.T, bundle []byte) string { - t.Helper() - dir := t.TempDir() - payload, err := os.MkdirTemp(dir, "..payload-") - if err != nil { - t.Fatalf("create payload dir: %v", err) - } - if err := os.WriteFile(filepath.Join(payload, "bundle.pem"), bundle, 0o600); err != nil { - t.Fatalf("write payload: %v", err) - } - if err := os.Symlink(filepath.Base(payload), filepath.Join(dir, "..data")); err != nil { - t.Fatalf("symlink ..data: %v", err) - } - if err := os.Symlink(filepath.Join("..data", "bundle.pem"), filepath.Join(dir, "bundle.pem")); err != nil { - t.Fatalf("symlink bundle.pem: %v", err) - } - return filepath.Join(dir, "bundle.pem") -} - -// rotateProjectedBundle replaces the bundle behind a writeProjectedBundle path -// as the kubelet does: a ..data symlink to a fresh payload directory is renamed -// over the old one, and the old payload removed. -func rotateProjectedBundle(path string, bundle []byte) error { - dir, name := filepath.Dir(path), filepath.Base(path) - oldPayload, err := os.Readlink(filepath.Join(dir, "..data")) - if err != nil { - return fmt.Errorf("readlink ..data: %w", err) - } - payload, err := os.MkdirTemp(dir, "..payload-") - if err != nil { - return fmt.Errorf("create payload dir: %w", err) - } - if err := os.WriteFile(filepath.Join(payload, name), bundle, 0o600); err != nil { - return fmt.Errorf("write payload: %w", err) - } - tmpLink := filepath.Join(dir, "..data_tmp") - if err := os.Symlink(filepath.Base(payload), tmpLink); err != nil { - return fmt.Errorf("symlink ..data_tmp: %w", err) - } - if err := os.Rename(tmpLink, filepath.Join(dir, "..data")); err != nil { - return fmt.Errorf("swap ..data: %w", err) - } - if err := os.RemoveAll(filepath.Join(dir, oldPayload)); err != nil { - return fmt.Errorf("remove old payload dir: %w", err) - } - return nil -} diff --git a/internal/plugins/gcp-secret-manager/internal/mtls/mtls.go b/internal/plugins/gcp-secret-manager/internal/mtls/mtls.go deleted file mode 100644 index e146262381..0000000000 --- a/internal/plugins/gcp-secret-manager/internal/mtls/mtls.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package mtls builds the provider's mutual-TLS server credentials. A caller -// must chain to the client trust bundle and carry the expected URI SAN, so no -// other workload that CA trusts can fetch secrets. Both files reload on -// rotation without a restart. -package mtls - -import ( - "crypto/tls" - "errors" - "fmt" - - "google.golang.org/grpc/credentials" - - "go.ate.dev/substrate/internal/plugins/gcp-secret-manager/internal/credbundle" -) - -// Config names the files and caller identity ServerCredentials serves with. -type Config struct { - // ServerBundle is the serving credential bundle (PKCS#8 key and chain). - ServerBundle string - // ClientCAFile is the trust bundle a caller's certificate must chain to. - ClientCAFile string - // CallerIdentity is the URI SAN a caller's certificate must carry. - CallerIdentity string -} - -// ServerCredentials returns gRPC server credentials requiring TLS 1.3 and a -// client certificate satisfying cfg. It fails if a field is empty or either -// bundle is unreadable, so a missing projection fails startup rather than -// leaving a Ready pod that refuses every connection. -func ServerCredentials(cfg Config) (credentials.TransportCredentials, error) { - switch { - case cfg.ServerBundle == "": - return nil, errors.New("a server credential bundle is required") - case cfg.ClientCAFile == "": - return nil, errors.New("a client trust bundle is required") - case cfg.CallerIdentity == "": - return nil, errors.New("a caller identity is required") - } - - loadPool := credbundle.PoolLoader(cfg.ClientCAFile) - if _, err := loadPool(); err != nil { - return nil, err - } - serverCert := credbundle.Loader(cfg.ServerBundle) - if _, err := serverCert(nil); err != nil { - return nil, err - } - verifySAN := verifyCallerSAN(cfg.CallerIdentity) - - // A per-connection config picks up a rotated client CA without a restart. - return credentials.NewTLS(&tls.Config{ - GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { - pool, err := loadPool() - if err != nil { - return nil, err - } - return &tls.Config{ - MinVersion: tls.VersionTLS13, - GetCertificate: serverCert, - ClientAuth: tls.RequireAndVerifyClientCert, - ClientCAs: pool, - VerifyConnection: verifySAN, - }, nil - }, - }), nil -} - -// verifyCallerSAN returns a VerifyConnection callback requiring expectedSAN -// among the caller certificate's URI SANs. -func verifyCallerSAN(expectedSAN string) func(tls.ConnectionState) error { - return func(state tls.ConnectionState) error { - if len(state.PeerCertificates) == 0 { - return errors.New("client certificate is required") - } - leaf := state.PeerCertificates[0] - for _, u := range leaf.URIs { - if u.String() == expectedSAN { - return nil - } - } - return fmt.Errorf("client certificate URI SANs %v do not include the expected caller identity %q", leaf.URIs, expectedSAN) - } -} diff --git a/internal/plugins/gcp-secret-manager/internal/mtls/mtls_test.go b/internal/plugins/gcp-secret-manager/internal/mtls/mtls_test.go deleted file mode 100644 index 8404c019c4..0000000000 --- a/internal/plugins/gcp-secret-manager/internal/mtls/mtls_test.go +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mtls - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "errors" - "io/fs" - "math/big" - "net" - "net/url" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "google.golang.org/grpc/credentials" -) - -// testCaller is a caller identity shaped like the egress gateway's. -const testCaller = "spiffe://cluster.local/ns/ate-system/sa/atenet-egress" - -func certWithURIs(t *testing.T, uris ...string) *x509.Certificate { - t.Helper() - cert := &x509.Certificate{} - for _, u := range uris { - parsed, err := url.Parse(u) - if err != nil { - t.Fatalf("parsing SAN %q: %v", u, err) - } - cert.URIs = append(cert.URIs, parsed) - } - return cert -} - -func TestVerifyCallerSAN(t *testing.T) { - tests := []struct { - name string - state tls.ConnectionState - wantErr bool - }{ - { - name: "matching SAN", - state: tls.ConnectionState{PeerCertificates: []*x509.Certificate{certWithURIs(t, testCaller)}}, - }, - { - name: "matching SAN among several", - state: tls.ConnectionState{PeerCertificates: []*x509.Certificate{certWithURIs(t, "spiffe://cluster.local/ns/other/sa/x", testCaller)}}, - }, - { - name: "wrong SAN", - state: tls.ConnectionState{PeerCertificates: []*x509.Certificate{certWithURIs(t, "spiffe://cluster.local/ns/ate-system/sa/impostor")}}, - wantErr: true, - }, - { - name: "no URI SANs", - state: tls.ConnectionState{PeerCertificates: []*x509.Certificate{certWithURIs(t)}}, - wantErr: true, - }, - { - name: "no peer certificate", - state: tls.ConnectionState{}, - wantErr: true, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := verifyCallerSAN(testCaller)(tc.state) - if tc.wantErr && err == nil { - t.Fatal("expected an error, got nil") - } - if !tc.wantErr && err != nil { - t.Fatalf("unexpected error: %v", err) - } - }) - } -} - -func TestServerCredentialsRequiresConfig(t *testing.T) { - caFile := filepath.Join(t.TempDir(), "client-ca.pem") - writeFileWithMtime(t, caFile, newCA(t, "client-ca").certPEM, time.Now()) - bundle := writeCredBundle(t, newCA(t, "server-ca").issue(t, certOpts{dnsNames: []string{"credprovider.test"}})) - valid := Config{ServerBundle: bundle, ClientCAFile: caFile, CallerIdentity: testCaller} - absent := filepath.Join(t.TempDir(), "absent.pem") - - tests := []struct { - name string - mutate func(*Config) - // wantIs, when set, must be in the error's chain. - wantIs error - }{ - {name: "no server bundle", mutate: func(c *Config) { c.ServerBundle = "" }}, - {name: "no client CA file", mutate: func(c *Config) { c.ClientCAFile = "" }}, - {name: "no caller identity", mutate: func(c *Config) { c.CallerIdentity = "" }}, - {name: "client CA file missing", mutate: func(c *Config) { c.ClientCAFile = absent }, wantIs: fs.ErrNotExist}, - {name: "server bundle missing", mutate: func(c *Config) { c.ServerBundle = absent }, wantIs: fs.ErrNotExist}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cfg := valid - tc.mutate(&cfg) - _, err := ServerCredentials(cfg) - if err == nil { - t.Fatal("ServerCredentials() error = nil, want an error") - } - if tc.wantIs != nil && !errors.Is(err, tc.wantIs) { - t.Errorf("ServerCredentials() error = %v, want one wrapping %v", err, tc.wantIs) - } - }) - } - if _, err := ServerCredentials(valid); err != nil { - t.Fatalf("ServerCredentials(valid) error = %v", err) - } -} - -// A rotated client trust bundle takes effect on the next connection without -// rebuilding the credentials, and the chain and SAN checks still hold. -func TestServerCredentialsReloadsClientCA(t *testing.T) { - serverCA := newCA(t, "server-ca") - serverBundlePath := writeCredBundle(t, serverCA.issue(t, certOpts{dnsNames: []string{"credprovider.test"}})) - serverRoots := x509.NewCertPool() - serverRoots.AddCert(serverCA.cert) - - clientCA1 := newCA(t, "client-ca-1") - clientCA2 := newCA(t, "client-ca-2") - - caFile := filepath.Join(t.TempDir(), "client-ca.pem") - writeFileWithMtime(t, caFile, clientCA1.certPEM, time.Now()) - - creds, err := ServerCredentials(Config{ - ServerBundle: serverBundlePath, - ClientCAFile: caFile, - CallerIdentity: testCaller, - }) - if err != nil { - t.Fatalf("ServerCredentials() error = %v", err) - } - - fromCA1 := clientCA1.issue(t, certOpts{uris: []string{testCaller}}) - fromCA2 := clientCA2.issue(t, certOpts{uris: []string{testCaller}}) - wrongSAN := clientCA1.issue(t, certOpts{uris: []string{"spiffe://cluster.local/ns/ate-system/sa/impostor"}}) - - // Before rotation only CA1 is trusted. - if err := handshake(t, creds, serverRoots, fromCA1); err != nil { - t.Fatalf("handshake with CA1-signed client cert failed before rotation: %v", err) - } - if err := handshake(t, creds, serverRoots, fromCA2); !isUnknownAuthority(err) { - t.Fatalf("handshake with CA2-signed client cert before rotation: err = %v, want an unknown-authority failure", err) - } - // A valid chain with the wrong SAN is rejected. - if err := handshake(t, creds, serverRoots, wrongSAN); err == nil || !strings.Contains(err.Error(), "do not include the expected caller identity") { - t.Fatalf("handshake with wrong-SAN client cert: err = %v, want the caller identity check to refuse it", err) - } - - // Rotate to CA2, bumping the mtime past coarse timestamps. - writeFileWithMtime(t, caFile, clientCA2.certPEM, time.Now().Add(time.Second)) - - // The same credentials now trust CA2 and not CA1. - if err := handshake(t, creds, serverRoots, fromCA2); err != nil { - t.Fatalf("handshake with CA2-signed client cert failed after rotation: %v", err) - } - if err := handshake(t, creds, serverRoots, fromCA1); !isUnknownAuthority(err) { - t.Fatalf("handshake with CA1-signed client cert after rotation: err = %v, want an unknown-authority failure", err) - } -} - -// isUnknownAuthority reports whether err is an untrusted-chain failure. -func isUnknownAuthority(err error) bool { - var unknown x509.UnknownAuthorityError - return errors.As(err, &unknown) -} - -// The server refuses a TLS 1.2-only client and one with no certificate. -func TestServerCredentialsRejectsWeakClients(t *testing.T) { - serverCA := newCA(t, "server-ca") - serverRoots := x509.NewCertPool() - serverRoots.AddCert(serverCA.cert) - clientCA := newCA(t, "client-ca") - caFile := filepath.Join(t.TempDir(), "client-ca.pem") - writeFileWithMtime(t, caFile, clientCA.certPEM, time.Now()) - creds, err := ServerCredentials(Config{ - ServerBundle: writeCredBundle(t, serverCA.issue(t, certOpts{dnsNames: []string{"credprovider.test"}})), - ClientCAFile: caFile, - CallerIdentity: testCaller, - }) - if err != nil { - t.Fatalf("ServerCredentials() error = %v", err) - } - caller := clientCA.issue(t, certOpts{uris: []string{testCaller}}) - - for _, tc := range []struct { - name string - mutate func(*tls.Config) - wantErr string - }{ - { - name: "TLS 1.2 only", - mutate: func(c *tls.Config) { c.MinVersion, c.MaxVersion = tls.VersionTLS12, tls.VersionTLS12 }, - wantErr: "unsupported versions", - }, - { - name: "no client certificate", - mutate: func(c *tls.Config) { - c.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { - return &tls.Certificate{}, nil - } - }, - wantErr: "didn't provide a certificate", - }, - } { - t.Run(tc.name, func(t *testing.T) { - cfg := clientConfig(serverRoots, caller) - tc.mutate(cfg) - if err := handshakeWith(t, creds, cfg); err == nil || !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("handshake err = %v, want one mentioning %q", err, tc.wantErr) - } - }) - } -} - -// handshake runs handshakeWith presenting clientCert and trusting serverRoots. -func handshake(t *testing.T, creds credentials.TransportCredentials, serverRoots *x509.CertPool, clientCert issued) error { - t.Helper() - return handshakeWith(t, creds, clientConfig(serverRoots, clientCert)) -} - -// clientConfig returns a client TLS config that always presents clientCert. With -// Certificates instead, Go sends no certificate unless the server names its -// issuer, so an untrusted-CA test would never reach chain verification. -func clientConfig(serverRoots *x509.CertPool, clientCert issued) *tls.Config { - cert := &tls.Certificate{Certificate: [][]byte{clientCert.certDER}, PrivateKey: clientCert.key} - return &tls.Config{ - MinVersion: tls.VersionTLS13, - RootCAs: serverRoots, - ServerName: "credprovider.test", - GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { - return cert, nil - }, - // The gRPC server credentials enforce ALPN, so offer "h2". - NextProtos: []string{"h2"}, - } -} - -// handshakeWith runs one TLS handshake against creds with clientCfg, returning -// the server's error if any, since refusals are decided there, else the -// client's. -func handshakeWith(t *testing.T, creds credentials.TransportCredentials, clientCfg *tls.Config) error { - t.Helper() - - lis, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatalf("listen: %v", err) - } - defer lis.Close() - - serverErr := make(chan error, 1) - go func() { - conn, err := lis.Accept() - if err != nil { - serverErr <- err - return - } - defer conn.Close() - _, _, err = creds.ServerHandshake(conn) - serverErr <- err - }() - - conn, clientErr := tls.Dial("tcp", lis.Addr().String(), clientCfg) - if clientErr == nil { - clientErr = conn.Handshake() - conn.Close() - } - - if err := <-serverErr; err != nil { - return err - } - return clientErr -} - -// ca is a self-signed certificate authority used to issue test certificates. -type ca struct { - cert *x509.Certificate - key *ecdsa.PrivateKey - certPEM []byte -} - -func newCA(t *testing.T, cn string) *ca { - t.Helper() - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - t.Fatalf("generate CA key: %v", err) - } - tmpl := &x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{CommonName: cn}, - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(time.Hour), - KeyUsage: x509.KeyUsageCertSign, - BasicConstraintsValid: true, - IsCA: true, - } - der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) - if err != nil { - t.Fatalf("create CA certificate: %v", err) - } - cert, err := x509.ParseCertificate(der) - if err != nil { - t.Fatalf("parse CA certificate: %v", err) - } - return &ca{cert: cert, key: key, certPEM: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})} -} - -type certOpts struct { - dnsNames []string - uris []string -} - -// issued is a leaf certificate and its private key. -type issued struct { - certDER []byte - key *ecdsa.PrivateKey -} - -func (c *ca) issue(t *testing.T, opts certOpts) issued { - t.Helper() - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - t.Fatalf("generate leaf key: %v", err) - } - var uris []*url.URL - for _, u := range opts.uris { - parsed, err := url.Parse(u) - if err != nil { - t.Fatalf("parse URI SAN %q: %v", u, err) - } - uris = append(uris, parsed) - } - tmpl := &x509.Certificate{ - SerialNumber: big.NewInt(time.Now().UnixNano()), - Subject: pkix.Name{CommonName: "leaf"}, - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(time.Hour), - KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - DNSNames: opts.dnsNames, - URIs: uris, - } - der, err := x509.CreateCertificate(rand.Reader, tmpl, c.cert, &key.PublicKey, c.key) - if err != nil { - t.Fatalf("create leaf certificate: %v", err) - } - return issued{certDER: der, key: key} -} - -// writeCredBundle writes leaf as a credential bundle and returns its path. -func writeCredBundle(t *testing.T, leaf issued) string { - t.Helper() - keyDER, err := x509.MarshalPKCS8PrivateKey(leaf.key) - if err != nil { - t.Fatalf("marshal PKCS8 key: %v", err) - } - bundle := append( - pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leaf.certDER}), - pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})..., - ) - path := filepath.Join(t.TempDir(), "server-bundle.pem") - if err := os.WriteFile(path, bundle, 0o600); err != nil { - t.Fatalf("write credential bundle: %v", err) - } - return path -} - -func writeFileWithMtime(t *testing.T, path string, data []byte, mtime time.Time) { - t.Helper() - if err := os.WriteFile(path, data, 0o600); err != nil { - t.Fatalf("write %s: %v", path, err) - } - if err := os.Chtimes(path, mtime, mtime); err != nil { - t.Fatalf("chtimes %s: %v", path, err) - } -} diff --git a/internal/plugins/gcp-secret-manager/internal/provider/projectauthz.go b/internal/plugins/gcp-secret-manager/internal/provider/projectauthz.go deleted file mode 100644 index 8106df20ee..0000000000 --- a/internal/plugins/gcp-secret-manager/internal/provider/projectauthz.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package provider - -import ( - "fmt" - "net/url" - "os" - "regexp" - "strings" - - "sigs.k8s.io/yaml" -) - -// projectPolicyFile is the policy YAML: grants mapping an atespace to the -// projects whose secrets it may resolve. -type projectPolicyFile struct { - Policies []atespaceProjectPolicy `json:"policies"` -} - -type atespaceProjectPolicy struct { - Atespace string `json:"atespace"` - AllowedProjects []string `json:"allowedProjects"` -} - -// ProjectAuthorizer decides which Google Cloud projects an atespace may resolve -// secrets in. It is default-deny; a nil authorizer refuses everything. Projects -// match as written: a grant by ID does not cover a URI naming it by number. -type ProjectAuthorizer struct { - // allowed maps atespace -> set of permitted projects. - allowed map[string]map[string]struct{} -} - -// LoadProjectAuthorizer loads the YAML policy at path. Unknown fields are -// errors, so a misspelled grant fails loudly rather than granting nothing. -func LoadProjectAuthorizer(path string) (*ProjectAuthorizer, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("reading project policy file %q: %w", path, err) - } - var file projectPolicyFile - if err := yaml.UnmarshalStrict(data, &file); err != nil { - return nil, fmt.Errorf("parsing project policy file %q: %w", path, err) - } - return newProjectAuthorizer(file) -} - -// newProjectAuthorizer validates file and builds an authorizer from it. -func newProjectAuthorizer(file projectPolicyFile) (*ProjectAuthorizer, error) { - allowed := make(map[string]map[string]struct{}) - for i, p := range file.Policies { - if p.Atespace == "" { - return nil, fmt.Errorf("project policy %d: atespace is required", i) - } - set := allowed[p.Atespace] - if set == nil { - set = make(map[string]struct{}) - allowed[p.Atespace] = set - } - for _, project := range p.AllowedProjects { - if project == "" || strings.Contains(project, "/") { - return nil, fmt.Errorf("project policy %d: allowed project %q must be a project ID or number", i, project) - } - set[project] = struct{}{} - } - } - return &ProjectAuthorizer{allowed: allowed}, nil -} - -// Allowed reports whether atespace may resolve secrets in project. -func (a *ProjectAuthorizer) Allowed(atespace, project string) bool { - if a == nil { - return false - } - _, ok := a.allowed[atespace][project] - return ok -} - -// actorTrustDomain is the trust domain of the actor SPIFFE IDs substrate mints. -const actorTrustDomain = "substrate-actor.local" - -// dns1123Label matches a substrate resource name, atespaces included. -var dns1123Label = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$`) - -// actorAtespace returns the atespace of an actor SPIFFE ID, -// spiffe://substrate-actor.local/actor//. It follows substrate's -// internal/resources.ActorRefFromActorSPIFFEID, which this module cannot import. -func actorAtespace(id string) (string, error) { - const format = "spiffe://" + actorTrustDomain + "/actor//" - u, err := url.Parse(id) - if err != nil { - return "", fmt.Errorf("invalid actor SPIFFE ID %q: %w", id, err) - } - if u.Scheme != "spiffe" || u.Host != actorTrustDomain || u.User != nil || u.RawQuery != "" || u.Fragment != "" { - return "", fmt.Errorf("%q does not have format %s", id, format) - } - segments := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") - if len(segments) != 3 || segments[0] != "actor" { - return "", fmt.Errorf("%q does not have format %s", id, format) - } - atespace, name := segments[1], segments[2] - if !dns1123Label.MatchString(atespace) { - return "", fmt.Errorf("%q is not a valid atespace", atespace) - } - if !dns1123Label.MatchString(name) { - return "", fmt.Errorf("%q is not a valid actor name", name) - } - return atespace, nil -} diff --git a/internal/plugins/gcp-secret-manager/internal/provider/projectauthz_test.go b/internal/plugins/gcp-secret-manager/internal/provider/projectauthz_test.go deleted file mode 100644 index 68b79ba615..0000000000 --- a/internal/plugins/gcp-secret-manager/internal/provider/projectauthz_test.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package provider - -import ( - "os" - "path/filepath" - "testing" - - "sigs.k8s.io/yaml" -) - -func TestLoadProjectAuthorizer(t *testing.T) { - for _, tc := range []struct { - name string - policy string - wantErr bool - }{ - {name: "empty grants nothing", policy: "policies: []\n"}, - {name: "grants", policy: "policies:\n- atespace: team-a\n allowedProjects: [proj-123, \"123456789\"]\n"}, - {name: "unknown field", policy: "policies:\n- atespace: team-a\n allowedProject: [proj-123]\n", wantErr: true}, - {name: "missing atespace", policy: "policies:\n- allowedProjects: [proj-123]\n", wantErr: true}, - {name: "empty project", policy: "policies:\n- atespace: team-a\n allowedProjects: [\"\"]\n", wantErr: true}, - {name: "project as a resource name", policy: "policies:\n- atespace: team-a\n allowedProjects: [projects/proj-123]\n", wantErr: true}, - {name: "not YAML", policy: "policies: [\n", wantErr: true}, - } { - t.Run(tc.name, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "project-policy.yaml") - if err := os.WriteFile(path, []byte(tc.policy), 0o600); err != nil { - t.Fatalf("writing the policy: %v", err) - } - _, err := LoadProjectAuthorizer(path) - if (err != nil) != tc.wantErr { - t.Errorf("LoadProjectAuthorizer() error = %v, wantErr %v", err, tc.wantErr) - } - }) - } - - if _, err := LoadProjectAuthorizer(filepath.Join(t.TempDir(), "absent.yaml")); err == nil { - t.Error("LoadProjectAuthorizer() of a missing file succeeded, want an error") - } -} - -func TestProjectAuthorizerAllowed(t *testing.T) { - a, err := newProjectAuthorizer(projectPolicyFile{Policies: []atespaceProjectPolicy{ - {Atespace: "team-a", AllowedProjects: []string{"proj-123"}}, - // Grants for one atespace accumulate across entries. - {Atespace: "team-a", AllowedProjects: []string{"proj-456"}}, - {Atespace: "team-b", AllowedProjects: []string{"123456789"}}, - {Atespace: "team-c"}, - }}) - if err != nil { - t.Fatalf("newProjectAuthorizer: %v", err) - } - - for _, tc := range []struct { - atespace, project string - want bool - }{ - {"team-a", "proj-123", true}, - {"team-a", "proj-456", true}, - {"team-a", "proj-789", false}, - {"team-b", "123456789", true}, - // Matched as written: a project granted by number is not granted by ID. - {"team-b", "proj-123", false}, - {"team-c", "proj-123", false}, - {"team-d", "proj-123", false}, - } { - if got := a.Allowed(tc.atespace, tc.project); got != tc.want { - t.Errorf("Allowed(%q, %q) = %v, want %v", tc.atespace, tc.project, got, tc.want) - } - } - - var none *ProjectAuthorizer - if none.Allowed("team-a", "proj-123") { - t.Error("a nil authorizer allowed a project, want it to allow nothing") - } -} - -// The shipped sample policy loads and grants what its comment says. -func TestShippedProjectPolicyLoads(t *testing.T) { - raw, err := os.ReadFile(filepath.Join("..", "..", "config", "project-policy.yaml")) - if err != nil { - t.Fatalf("reading the shipped policy: %v", err) - } - var cm struct { - Data map[string]string `json:"data"` - } - if err := yaml.Unmarshal(raw, &cm); err != nil { - t.Fatalf("parsing the shipped policy's ConfigMap: %v", err) - } - path := filepath.Join(t.TempDir(), "project-policy.yaml") - if err := os.WriteFile(path, []byte(cm.Data["project-policy.yaml"]), 0o600); err != nil { - t.Fatalf("writing the policy: %v", err) - } - a, err := LoadProjectAuthorizer(path) - if err != nil { - t.Fatalf("LoadProjectAuthorizer: %v", err) - } - if !a.Allowed("team-a", "my-project") { - t.Error("the shipped policy does not grant team-a my-project, as its comment says") - } -} - -func TestActorAtespace(t *testing.T) { - for _, tc := range []struct { - id string - want string - wantErr bool - }{ - {id: "spiffe://substrate-actor.local/actor/team-a/agent-1", want: "team-a"}, - {id: "", wantErr: true}, - {id: "spiffe://cluster.local/actor/team-a/agent-1", wantErr: true}, - {id: "spiffe://substrate-actor.local/worker-for-actor/team-a/agent-1", wantErr: true}, - {id: "spiffe://substrate-actor.local/actor/team-a", wantErr: true}, - {id: "spiffe://substrate-actor.local/actor/team-a/agent-1/extra", wantErr: true}, - {id: "spiffe://substrate-actor.local/actor/Team_A/agent-1", wantErr: true}, - {id: "spiffe://substrate-actor.local/actor/team-a/agent-1?x=y", wantErr: true}, - {id: "spiffe://user@substrate-actor.local/actor/team-a/agent-1", wantErr: true}, - {id: "https://substrate-actor.local/actor/team-a/agent-1", wantErr: true}, - } { - got, err := actorAtespace(tc.id) - if (err != nil) != tc.wantErr { - t.Errorf("actorAtespace(%q) error = %v, wantErr %v", tc.id, err, tc.wantErr) - continue - } - if got != tc.want { - t.Errorf("actorAtespace(%q) = %q, want %q", tc.id, got, tc.want) - } - } -} diff --git a/internal/plugins/gcp-secret-manager/internal/provider/provider.go b/internal/plugins/gcp-secret-manager/internal/provider/provider.go deleted file mode 100644 index ef9e7d830b..0000000000 --- a/internal/plugins/gcp-secret-manager/internal/provider/provider.go +++ /dev/null @@ -1,328 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package provider implements substrate's CredentialProvider API over Google -// Cloud Secret Manager, resolving ate-secret:// URIs to a secret version's -// payload or to one key's value in a JSON payload. Substrate never stores the -// secret. -package provider - -import ( - "context" - "encoding/json" - "fmt" - "hash/crc32" - "log/slog" - "net/url" - "regexp" - "strings" - "time" - - secretmanagerpb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" - "github.com/googleapis/gax-go/v2" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "go.ate.dev/substrate/api/credprovider/v1" -) - -// providerName is the ate-secret:// URI host this backend serves. -const providerName = "secretmanager.googleapis.com" - -// uriScheme is the only scheme a credential URI may carry. -const uriScheme = "ate-secret" - -// latestVersion is the alias for a secret's newest version. Secret Manager -// refuses it with FailedPrecondition while that version is disabled. -const latestVersion = "latest" - -// Name formats, checked so a malformed name fails as InvalidArgument rather -// than as whatever Secret Manager makes of it. -var ( - projectID = regexp.MustCompile(`^[a-z][a-z0-9-]{4,28}[a-z0-9]$`) - projectNumber = regexp.MustCompile(`^[0-9]+$`) - secretID = regexp.MustCompile(`^[A-Za-z0-9_-]{1,255}$`) -) - -// region matches a Google Cloud region such as us-central1. The location -// becomes part of the regional endpoint's hostname, so it must not admit a dot -// or anything else that could change the host. -var region = regexp.MustCompile(`^[a-z]+-[a-z]+[0-9]+$`) - -// crc32cTable is the polynomial Secret Manager checksums payloads with. -var crc32cTable = crc32.MakeTable(crc32.Castagnoli) - -// secretRef is a parsed ate-secret:// URI: a secret version's resource name, -// optionally followed by a key into its JSON payload. -// -// ate-secret://secretmanager.googleapis.com/projects//secrets//versions/ -// ate-secret://secretmanager.googleapis.com/projects//secrets//versions//keys/ -// -// A regional secret adds locations/ after the project, as its -// resource name does. -type secretRef struct { - // Project is the Google Cloud project ID or number owning the secret. - Project string - // Location is the region of a regional secret; empty for a global one. - Location string - // Secret is the Secret Manager secret ID. - Secret string - // Version is a positive integer or "latest". - Version string - // Key, if set, is the top-level JSON key whose value is the credential. - Key string -} - -// resourceName is the Secret Manager resource name of the referenced version. -func (r secretRef) resourceName() string { - if r.Location != "" { - return fmt.Sprintf("projects/%s/locations/%s/secrets/%s/versions/%s", r.Project, r.Location, r.Secret, r.Version) - } - return fmt.Sprintf("projects/%s/secrets/%s/versions/%s", r.Project, r.Secret, r.Version) -} - -// parseURI parses a secretmanager.googleapis.com ate-secret:// URI, rejecting -// any other scheme or provider and any malformed resource name. -func parseURI(raw string) (secretRef, error) { - u, err := url.Parse(raw) - if err != nil { - return secretRef{}, fmt.Errorf("parsing credential URI %q: %w", raw, err) - } - if u.Scheme != uriScheme { - return secretRef{}, fmt.Errorf("malformed credential URI %q: scheme is %q, want %q", raw, u.Scheme, uriScheme) - } - if u.Host != providerName { - return secretRef{}, fmt.Errorf("credential URI %q: provider is %q, this provider serves %q", raw, u.Host, providerName) - } - // Reject, rather than ignore, components the grammar does not define. - if u.User != nil { - return secretRef{}, fmt.Errorf("credential URI %q: user information is not allowed", raw) - } - if u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { - return secretRef{}, fmt.Errorf("credential URI %q: query and fragment components are not allowed", raw) - } - // A Secret Manager resource name never needs percent-encoding. - if u.EscapedPath() != u.Path { - return secretRef{}, fmt.Errorf("credential URI %q: path must not contain percent-encoding", raw) - } - - const want = "projects/[/locations/]/secrets//versions/[/keys/]" - // Split without trimming, so an extra slash becomes an empty segment. - if !strings.HasPrefix(u.Path, "/") { - return secretRef{}, fmt.Errorf("credential URI %q: want %s, got path %q", raw, want, u.Path) - } - segments := strings.Split(u.Path[1:], "/") - for i, seg := range segments { - if seg == "" { - return secretRef{}, fmt.Errorf("credential URI %q: empty path segment %d", raw, i) - } - } - if len(segments) < 2 || segments[0] != "projects" { - return secretRef{}, fmt.Errorf("credential URI %q: want %s, got path %q", raw, want, u.Path) - } - if !projectID.MatchString(segments[1]) && !projectNumber.MatchString(segments[1]) { - return secretRef{}, fmt.Errorf("credential URI %q: project %q must be a project ID or number", raw, segments[1]) - } - ref := secretRef{Project: segments[1]} - rest := segments[2:] - - if len(rest) >= 2 && rest[0] == "locations" { - if !region.MatchString(rest[1]) { - return secretRef{}, fmt.Errorf("credential URI %q: location %q must be a region, such as us-central1", raw, rest[1]) - } - ref.Location = rest[1] - rest = rest[2:] - } - - if (len(rest) != 4 && len(rest) != 6) || rest[0] != "secrets" || rest[2] != "versions" { - return secretRef{}, fmt.Errorf("credential URI %q: want %s, got path %q", raw, want, u.Path) - } - if !secretID.MatchString(rest[1]) { - return secretRef{}, fmt.Errorf("credential URI %q: secret ID %q must be 1-255 letters, digits, underscores or dashes", raw, rest[1]) - } - if !validVersion(rest[3]) { - return secretRef{}, fmt.Errorf("credential URI %q: version %q must be a positive integer or %q", raw, rest[3], latestVersion) - } - ref.Secret, ref.Version = rest[1], rest[3] - if len(rest) == 6 { - if rest[4] != "keys" { - return secretRef{}, fmt.Errorf("credential URI %q: want %s, got path %q", raw, want, u.Path) - } - // Dot segments are path traversal, never a key. - if rest[5] == "." || rest[5] == ".." { - return secretRef{}, fmt.Errorf("credential URI %q: key %q is not allowed", raw, rest[5]) - } - ref.Key = rest[5] - } - return ref, nil -} - -// validVersion reports whether v names a Secret Manager version: the "latest" -// alias or a positive integer without leading zeros. -func validVersion(v string) bool { - if v == latestVersion { - return true - } - if v == "" || v[0] == '0' { - return false - } - for _, c := range v { - if c < '0' || c > '9' { - return false - } - } - return true -} - -// SecretAccessor is the subset of *secretmanager.Client the server uses. -type SecretAccessor interface { - AccessSecretVersion(ctx context.Context, req *secretmanagerpb.AccessSecretVersionRequest, opts ...gax.CallOption) (*secretmanagerpb.AccessSecretVersionResponse, error) -} - -// Accessors returns the Secret Manager client for a location: "" for global, -// or a region. -type Accessors func(location string) (SecretAccessor, error) - -// Server implements credprovider.CredentialProviderServer over Secret -// Manager. mTLS admits only the egress gateway, which names each request's -// actor; a secret resolves only in a project the policy grants that actor's -// atespace. -type Server struct { - credprovider.UnimplementedCredentialProviderServer - - clients Accessors - authz *ProjectAuthorizer - fetchTimeout time.Duration -} - -// NewServer returns a Server that reads through clients the secrets in projects -// authz grants (none if authz is nil). Each Secret Manager call gets at most -// fetchTimeout, which must be positive. -func NewServer(clients Accessors, authz *ProjectAuthorizer, fetchTimeout time.Duration) *Server { - return &Server{clients: clients, authz: authz, fetchTimeout: fetchTimeout} -} - -// FetchSecret resolves an ate-secret:// URI to its secret version's payload, or -// to the string value of the URI's key in it, returned verbatim. -func (s *Server) FetchSecret(ctx context.Context, req *credprovider.FetchSecretRequest) (*credprovider.FetchSecretResponse, error) { - ref, err := parseURI(req.GetUri()) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - if err := s.authorize(ctx, req.GetActorSpiffeId(), ref.Project); err != nil { - return nil, err - } - - slog.InfoContext(ctx, "resolving credential", - slog.String("provider", providerName), - slog.String("project", ref.Project), - slog.String("location", ref.Location), - slog.String("secret", ref.Secret), - slog.String("version", ref.Version), - slog.String("key", ref.Key), - slog.String("actor", req.GetActorSpiffeId()), - ) - - // The gateway sets no deadline and the client retries for up to a minute; - // answer DeadlineExceeded before Envoy's ext_proc timeout abandons the call. - fetchCtx, cancel := context.WithTimeout(ctx, s.fetchTimeout) - defer cancel() - client, err := s.clients(ref.Location) - if err != nil { - return nil, status.Errorf(codes.Unavailable, "secret version %s: no Secret Manager client: %v", ref.resourceName(), err) - } - resp, err := client.AccessSecretVersion(fetchCtx, &secretmanagerpb.AccessSecretVersionRequest{Name: ref.resourceName()}) - if err != nil { - return nil, accessError(ref, err) - } - - payload := resp.GetPayload() - if payload == nil { - return nil, status.Errorf(codes.NotFound, "secret version %s has no payload", ref.resourceName()) - } - // Fail closed on a CRC32C mismatch rather than inject a corrupted credential. - if payload.DataCrc32C != nil { - if got := int64(crc32.Checksum(payload.GetData(), crc32cTable)); got != payload.GetDataCrc32C() { - return nil, status.Errorf(codes.Unavailable, "secret version %s: payload CRC32C mismatch", ref.resourceName()) - } - } - if ref.Key == "" { - return &credprovider.FetchSecretResponse{OpaqueBytes: payload.GetData()}, nil - } - value, err := keyValue(ref, payload.GetData()) - if err != nil { - return nil, err - } - return &credprovider.FetchSecretResponse{OpaqueBytes: value}, nil -} - -// authorize denies unless the policy grants project to the atespace of the -// actor SPIFFE ID the gateway asserts. -func (s *Server) authorize(ctx context.Context, actorSpiffeID, project string) error { - atespace, err := actorAtespace(actorSpiffeID) - if err != nil { - slog.WarnContext(ctx, "credential request denied: unusable actor identity", slog.Any("err", err)) - return status.Errorf(codes.PermissionDenied, "actor identity is required and must be a valid actor SPIFFE URI: %v", err) - } - if !s.authz.Allowed(atespace, project) { - slog.WarnContext(ctx, "credential request denied: atespace not permitted for project", - slog.String("atespace", atespace), slog.String("project", project)) - return status.Errorf(codes.PermissionDenied, "atespace %q is not permitted to resolve secrets in project %q", atespace, project) - } - return nil -} - -// accessError maps a Secret Manager error to a code for the egress gateway, -// which answers Unavailable and DeadlineExceeded with a retryable 503 and -// anything else with a 403. Failures a retry cannot fix keep a non-retryable -// code; unrecognized ones are treated as transient. -func accessError(ref secretRef, err error) error { - name := ref.resourceName() - msg := status.Convert(err).Message() - switch code := status.Code(err); code { - case codes.NotFound: - return status.Errorf(codes.NotFound, "secret version %s not found", name) - case codes.PermissionDenied, codes.Unauthenticated: - return status.Errorf(codes.PermissionDenied, "not permitted to access secret version %s: %s", name, msg) - case codes.FailedPrecondition, codes.InvalidArgument, codes.DeadlineExceeded: - return status.Errorf(code, "accessing secret version %s: %s", name, msg) - default: - return status.Errorf(codes.Unavailable, "accessing secret version %s: %s", name, msg) - } -} - -// keyValue returns the string value of ref.Key in the JSON object data. Only -// that value is decoded, so an unrepresentable sibling such as 1e999 does not -// fail it. Errors omit the decoder's message, which can quote the payload. -func keyValue(ref secretRef, data []byte) ([]byte, error) { - var obj map[string]json.RawMessage - // A top-level null decodes into a nil map without error. - if err := json.Unmarshal(data, &obj); err != nil || obj == nil { - return nil, status.Errorf(codes.FailedPrecondition, "secret version %s: payload is not a JSON object", ref.resourceName()) - } - raw, ok := obj[ref.Key] - if !ok { - return nil, status.Errorf(codes.NotFound, "secret version %s: payload has no key %q", ref.resourceName(), ref.Key) - } - // Into any rather than string: a string target takes null as a no-op. - var v any - if err := json.Unmarshal(raw, &v); err != nil { - return nil, status.Errorf(codes.FailedPrecondition, "secret version %s: value of key %q is not a JSON string", ref.resourceName(), ref.Key) - } - s, ok := v.(string) - if !ok { - return nil, status.Errorf(codes.FailedPrecondition, "secret version %s: value of key %q is not a JSON string", ref.resourceName(), ref.Key) - } - return []byte(s), nil -} diff --git a/internal/plugins/gcp-secret-manager/internal/provider/provider_test.go b/internal/plugins/gcp-secret-manager/internal/provider/provider_test.go deleted file mode 100644 index 95bc1be8dd..0000000000 --- a/internal/plugins/gcp-secret-manager/internal/provider/provider_test.go +++ /dev/null @@ -1,586 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package provider - -import ( - "context" - "errors" - "fmt" - "hash/crc32" - "strings" - "testing" - "time" - - secretmanagerpb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" - "github.com/googleapis/gax-go/v2" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "go.ate.dev/substrate/api/credprovider/v1" -) - -// fakeAccessor is a fake Secret Manager client serving payloads by resource -// name. -type fakeAccessor struct { - byName map[string][]byte - // errByName maps a resource name to an error to return instead. - errByName map[string]error - // corrupt sends a CRC32C that does not match the data. - corrupt bool - // noPayload omits the payload. - noPayload bool - // noCRC omits the CRC32C. - noCRC bool - // block answers only once the call's context is done. - block bool - // gotName is the last requested resource name. - gotName string -} - -func (f *fakeAccessor) AccessSecretVersion(ctx context.Context, req *secretmanagerpb.AccessSecretVersionRequest, _ ...gax.CallOption) (*secretmanagerpb.AccessSecretVersionResponse, error) { - f.gotName = req.GetName() - if f.block { - <-ctx.Done() - return nil, status.FromContextError(ctx.Err()).Err() - } - if err, ok := f.errByName[req.GetName()]; ok { - return nil, err - } - data, ok := f.byName[req.GetName()] - if !ok { - return nil, status.Errorf(codes.NotFound, "no such secret version") - } - if f.noPayload { - return &secretmanagerpb.AccessSecretVersionResponse{Name: req.GetName()}, nil - } - if f.noCRC { - return &secretmanagerpb.AccessSecretVersionResponse{ - Name: req.GetName(), - Payload: &secretmanagerpb.SecretPayload{Data: data}, - }, nil - } - crc := int64(crc32.Checksum(data, crc32cTable)) - if f.corrupt { - crc++ // deliberately wrong - } - return &secretmanagerpb.AccessSecretVersionResponse{ - Name: req.GetName(), - Payload: &secretmanagerpb.SecretPayload{Data: data, DataCrc32C: &crc}, - }, nil -} - -// only serves global secrets through a and fails for any region. -func only(a SecretAccessor) Accessors { - return func(location string) (SecretAccessor, error) { - if location != "" { - return nil, fmt.Errorf("no client for %q", location) - } - return a, nil - } -} - -func TestParseURI(t *testing.T) { - tests := []struct { - name string - uri string - want secretRef - wantErr bool - }{ - { - name: "with version", - uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/5", - want: secretRef{Project: "proj-123", Secret: "example-api", Version: "5"}, - }, - { - name: "latest alias", - uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/latest", - want: secretRef{Project: "proj-123", Secret: "example-api", Version: "latest"}, - }, - { - name: "with key", - uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/5/keys/api_key", - want: secretRef{Project: "proj-123", Secret: "example-api", Version: "5", Key: "api_key"}, - }, - { - name: "latest alias with key", - uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/latest/keys/token", - want: secretRef{Project: "proj-123", Secret: "example-api", Version: "latest", Key: "token"}, - }, - // The version is never implied; "latest" must be spelled out. - {name: "version required", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api", wantErr: true}, - {name: "version required with key", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/keys/token", wantErr: true}, - {name: "wrong scheme", uri: "https://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/1", wantErr: true}, - {name: "wrong provider", uri: "ate-secret://k8s.io/projects/proj-123/secrets/example-api/versions/1", wantErr: true}, - {name: "too few segments", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets", wantErr: true}, - {name: "wrong projects keyword", uri: "ate-secret://secretmanager.googleapis.com/project/proj-123/secrets/example-api/versions/1", wantErr: true}, - {name: "wrong secrets keyword", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secret/example-api/versions/1", wantErr: true}, - {name: "wrong versions keyword", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/version/5", wantErr: true}, - {name: "non-numeric version", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/newest", wantErr: true}, - {name: "zero version", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/0", wantErr: true}, - {name: "leading-zero version", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/05", wantErr: true}, - {name: "singular key keyword", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/s/versions/1/key/token", wantErr: true}, - {name: "keys keyword without a key", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/s/versions/1/keys", wantErr: true}, - {name: "too many segments", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/s/versions/1/keys/token/extra", wantErr: true}, - {name: "empty segment", uri: "ate-secret://secretmanager.googleapis.com/projects//secrets/example-api/versions/1", wantErr: true}, - {name: "query not allowed", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/1?key=token", wantErr: true}, - {name: "fragment not allowed", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/1#x", wantErr: true}, - {name: "percent-encoded separator", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example%2Fapi/versions/1", wantErr: true}, - {name: "percent-encoding of any kind", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example%2Dapi/versions/1", wantErr: true}, - {name: "percent-encoding in key", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/1/keys/api%2Dkey", wantErr: true}, - {name: "space in path", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example api/versions/1", wantErr: true}, - {name: "unparseable", uri: "://://", wantErr: true}, - { - name: "project number", - uri: "ate-secret://secretmanager.googleapis.com/projects/123456789012/secrets/example-api/versions/1", - want: secretRef{Project: "123456789012", Secret: "example-api", Version: "1"}, - }, - {name: "user information", uri: "ate-secret://user@secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/1", wantErr: true}, - {name: "doubled slash", uri: "ate-secret://secretmanager.googleapis.com/projects//proj-123/secrets/example-api/versions/1", wantErr: true}, - {name: "leading doubled slash", uri: "ate-secret://secretmanager.googleapis.com//projects/proj-123/secrets/example-api/versions/1", wantErr: true}, - {name: "trailing slash", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/1/", wantErr: true}, - {name: "no path", uri: "ate-secret://secretmanager.googleapis.com", wantErr: true}, - {name: "project with a character Secret Manager rejects", uri: "ate-secret://secretmanager.googleapis.com/projects/proj!123/secrets/example-api/versions/1", wantErr: true}, - {name: "project in upper case", uri: "ate-secret://secretmanager.googleapis.com/projects/Proj-123/secrets/example-api/versions/1", wantErr: true}, - {name: "project too short", uri: "ate-secret://secretmanager.googleapis.com/projects/p/secrets/example-api/versions/1", wantErr: true}, - {name: "project as a dot segment", uri: "ate-secret://secretmanager.googleapis.com/projects/../secrets/example-api/versions/1", wantErr: true}, - {name: "secret with a dot", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example.api/versions/1", wantErr: true}, - {name: "secret as a dot segment", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/../versions/1", wantErr: true}, - {name: "secret too long", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/" + strings.Repeat("s", 256) + "/versions/1", wantErr: true}, - { - name: "regional", - uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/locations/us-central1/secrets/example-api/versions/5", - want: secretRef{Project: "proj-123", Location: "us-central1", Secret: "example-api", Version: "5"}, - }, - { - name: "regional with key", - uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/locations/europe-west4/secrets/example-api/versions/latest/keys/token", - want: secretRef{Project: "proj-123", Location: "europe-west4", Secret: "example-api", Version: "latest", Key: "token"}, - }, - {name: "regional without a version", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/locations/us-central1/secrets/example-api", wantErr: true}, - {name: "locations keyword without a location", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/locations/secrets/example-api/versions/1", wantErr: true}, - {name: "locations keyword with nothing after", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/locations/us-central1", wantErr: true}, - // The location becomes part of an endpoint hostname. - {name: "location with a dot", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/locations/example.com/secrets/example-api/versions/1", wantErr: true}, - {name: "location in upper case", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/locations/US-CENTRAL1/secrets/example-api/versions/1", wantErr: true}, - {name: "location global", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/locations/global/secrets/example-api/versions/1", wantErr: true}, - {name: "regional with too many segments", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/locations/us-central1/secrets/example-api/versions/1/keys/token/extra", wantErr: true}, - {name: "key as a dot segment", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/1/keys/..", wantErr: true}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got, err := parseURI(tc.uri) - if tc.wantErr { - if err == nil { - t.Fatalf("parseURI(%q) = %+v, want error", tc.uri, got) - } - return - } - if err != nil { - t.Fatalf("parseURI(%q) unexpected error: %v", tc.uri, err) - } - if got != tc.want { - t.Errorf("parseURI(%q) = %+v, want %+v", tc.uri, got, tc.want) - } - }) - } -} - -func TestResourceName(t *testing.T) { - for _, tc := range []struct { - ref secretRef - want string - }{ - {secretRef{Project: "proj-123", Secret: "example-api", Version: "latest"}, "projects/proj-123/secrets/example-api/versions/latest"}, - {secretRef{Project: "proj-123", Location: "us-central1", Secret: "example-api", Version: "3"}, "projects/proj-123/locations/us-central1/secrets/example-api/versions/3"}, - } { - if got := tc.ref.resourceName(); got != tc.want { - t.Errorf("%+v.resourceName() = %q, want %q", tc.ref, got, tc.want) - } - } -} - -// testActor is an actor in the atespace testAuthorizer grants proj-123. -const testActor = "spiffe://substrate-actor.local/actor/team-a/agent-1" - -func testAuthorizer(t *testing.T) *ProjectAuthorizer { - t.Helper() - a, err := newProjectAuthorizer(projectPolicyFile{Policies: []atespaceProjectPolicy{ - {Atespace: "team-a", AllowedProjects: []string{"proj-123"}}, - }}) - if err != nil { - t.Fatalf("newProjectAuthorizer: %v", err) - } - return a -} - -func TestFetchSecret(t *testing.T) { - const secret = "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api" - const uri = secret + "/versions/latest" - const name = "projects/proj-123/secrets/example-api/versions/latest" - withPayload := func(payload string) *fakeAccessor { - return &fakeAccessor{byName: map[string][]byte{name: []byte(payload)}} - } - tests := []struct { - name string - accessor *fakeAccessor - uri string - want string - wantCode codes.Code - }{ - { - // Returned verbatim; the gateway trims and prefixes it. - name: "found", - accessor: &fakeAccessor{byName: map[string][]byte{name: []byte("s3cr3t\n")}}, - uri: uri, - want: "s3cr3t\n", - }, - { - name: "no checksum", - accessor: &fakeAccessor{byName: map[string][]byte{name: []byte("s3cr3t")}, noCRC: true}, - uri: uri, - want: "s3cr3t", - }, - { - // Only the selected value is decoded. - name: "key beside a value Go cannot represent", - accessor: withPayload(`{"api_key":"s3cr3t","n":1e999}`), - uri: uri + "/keys/api_key", - want: "s3cr3t", - }, - { - name: "key whose value Go cannot represent", - accessor: withPayload(`{"api_key":"s3cr3t","n":1e999}`), - uri: uri + "/keys/n", - wantCode: codes.FailedPrecondition, - }, - { - name: "numbered version", - accessor: &fakeAccessor{byName: map[string][]byte{"projects/proj-123/secrets/example-api/versions/5": []byte("v5")}}, - uri: secret + "/versions/5", - want: "v5", - }, - { - // Refused rather than read as "latest". - name: "version required", - accessor: &fakeAccessor{byName: map[string][]byte{name: []byte("s3cr3t")}}, - uri: secret, - wantCode: codes.InvalidArgument, - }, - { - name: "json payload without key", - accessor: withPayload(`{"api_key":"s3cr3t"}`), - uri: uri, - want: `{"api_key":"s3cr3t"}`, - }, - { - name: "key", - accessor: withPayload(`{"api_key":"s3cr3t","other":"x"}`), - uri: uri + "/keys/api_key", - want: "s3cr3t", - }, - { - name: "key value is unescaped", - accessor: withPayload(`{"token":"a\"b\n"}`), - uri: uri + "/keys/token", - want: "a\"b\n", - }, - { - name: "key missing is not found", - accessor: withPayload(`{"api_key":"s3cr3t"}`), - uri: uri + "/keys/token", - wantCode: codes.NotFound, - }, - { - name: "key on non-json payload", - accessor: withPayload("s3cr3t"), - uri: uri + "/keys/token", - wantCode: codes.FailedPrecondition, - }, - { - name: "key on json array", - accessor: withPayload(`["s3cr3t"]`), - uri: uri + "/keys/token", - wantCode: codes.FailedPrecondition, - }, - { - name: "key on json null", - accessor: withPayload("null"), - uri: uri + "/keys/token", - wantCode: codes.FailedPrecondition, - }, - { - name: "key value not a string", - accessor: withPayload(`{"token":12345}`), - uri: uri + "/keys/token", - wantCode: codes.FailedPrecondition, - }, - { - name: "key value null", - accessor: withPayload(`{"token":null}`), - uri: uri + "/keys/token", - wantCode: codes.FailedPrecondition, - }, - { - name: "key in nested object is not reached", - accessor: withPayload(`{"outer":{"token":"s3cr3t"}}`), - uri: uri + "/keys/token", - wantCode: codes.NotFound, - }, - { - name: "crc mismatch fails closed before key lookup", - accessor: &fakeAccessor{byName: map[string][]byte{name: []byte(`{"token":"s3cr3t"}`)}, corrupt: true}, - uri: uri + "/keys/token", - wantCode: codes.Unavailable, - }, - { - name: "not found", - accessor: &fakeAccessor{errByName: map[string]error{name: status.Error(codes.NotFound, "gone")}}, - uri: uri, - wantCode: codes.NotFound, - }, - { - name: "permission denied maps through", - accessor: &fakeAccessor{errByName: map[string]error{name: status.Error(codes.PermissionDenied, "nope")}}, - uri: uri, - wantCode: codes.PermissionDenied, - }, - // Only Unavailable and DeadlineExceeded are retryable at the gateway, so - // failures a retry cannot fix keep another code. - { - // A disabled or destroyed version, e.g. "latest" while the newest is disabled. - name: "failed precondition passes through", - accessor: &fakeAccessor{errByName: map[string]error{name: status.Error(codes.FailedPrecondition, "version is disabled")}}, - uri: uri, - wantCode: codes.FailedPrecondition, - }, - { - name: "invalid argument passes through", - accessor: &fakeAccessor{errByName: map[string]error{name: status.Error(codes.InvalidArgument, "bad name")}}, - uri: uri, - wantCode: codes.InvalidArgument, - }, - { - name: "deadline exceeded passes through", - accessor: &fakeAccessor{errByName: map[string]error{name: status.Error(codes.DeadlineExceeded, "slow")}}, - uri: uri, - wantCode: codes.DeadlineExceeded, - }, - { - // The provider's own credentials refused, e.g. misconfigured Workload Identity. - name: "unauthenticated is permission denied", - accessor: &fakeAccessor{errByName: map[string]error{name: status.Error(codes.Unauthenticated, "no credentials")}}, - uri: uri, - wantCode: codes.PermissionDenied, - }, - { - name: "internal error is unavailable", - accessor: &fakeAccessor{errByName: map[string]error{name: status.Error(codes.Internal, "boom")}}, - uri: uri, - wantCode: codes.Unavailable, - }, - { - name: "quota exhaustion is unavailable", - accessor: &fakeAccessor{errByName: map[string]error{name: status.Error(codes.ResourceExhausted, "quota")}}, - uri: uri, - wantCode: codes.Unavailable, - }, - { - name: "unavailable stays unavailable", - accessor: &fakeAccessor{errByName: map[string]error{name: status.Error(codes.Unavailable, "down")}}, - uri: uri, - wantCode: codes.Unavailable, - }, - { - // Not a gRPC status, e.g. a transport failure. - name: "unknown error is unavailable", - accessor: &fakeAccessor{errByName: map[string]error{name: errors.New("connection reset")}}, - uri: uri, - wantCode: codes.Unavailable, - }, - { - name: "no payload is not found", - accessor: &fakeAccessor{byName: map[string][]byte{name: []byte("s3cr3t")}, noPayload: true}, - uri: uri, - wantCode: codes.NotFound, - }, - { - name: "crc mismatch fails closed", - accessor: &fakeAccessor{byName: map[string][]byte{name: []byte("s3cr3t")}, corrupt: true}, - uri: uri, - wantCode: codes.Unavailable, - }, - { - name: "bad uri", - accessor: &fakeAccessor{}, - uri: "ate-secret://k8s.io/projects/proj-123/secrets/example-api", - wantCode: codes.InvalidArgument, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - srv := NewServer(only(tc.accessor), testAuthorizer(t), time.Second) - resp, err := srv.FetchSecret(context.Background(), &credprovider.FetchSecretRequest{Uri: tc.uri, ActorSpiffeId: testActor}) - if tc.wantCode != codes.OK { - if status.Code(err) != tc.wantCode { - t.Fatalf("FetchSecret(%q) code = %v, want %v (err=%v)", tc.uri, status.Code(err), tc.wantCode, err) - } - return - } - if err != nil { - t.Fatalf("FetchSecret(%q) unexpected error: %v", tc.uri, err) - } - if got := string(resp.GetOpaqueBytes()); got != tc.want { - t.Errorf("FetchSecret(%q) = %q, want %q", tc.uri, got, tc.want) - } - }) - } -} - -// A decode error omits the decoder's message, which quotes the secret. -func TestFetchSecretKeyErrorDoesNotQuotePayload(t *testing.T) { - const name = "projects/proj-123/secrets/example-api/versions/1" - accessor := &fakeAccessor{byName: map[string][]byte{name: []byte(`{"token": Xs3cr3t}`)}} - _, err := NewServer(only(accessor), testAuthorizer(t), time.Second).FetchSecret(context.Background(), &credprovider.FetchSecretRequest{ - Uri: "ate-secret://secretmanager.googleapis.com/" + name + "/keys/token", - ActorSpiffeId: testActor, - }) - if status.Code(err) != codes.FailedPrecondition { - t.Fatalf("code = %v, want FailedPrecondition (err=%v)", status.Code(err), err) - } - if msg := status.Convert(err).Message(); strings.Contains(msg, "X") { - t.Errorf("error %q quotes the payload", msg) - } -} - -// A URI that fails to parse is refused before Secret Manager is dialed. -func TestFetchSecretDoesNotDialOnBadURI(t *testing.T) { - accessor := &fakeAccessor{} - _, err := NewServer(only(accessor), testAuthorizer(t), time.Second).FetchSecret(context.Background(), &credprovider.FetchSecretRequest{ - Uri: "ate-secret://secretmanager.googleapis.com/nope", - ActorSpiffeId: testActor, - }) - if status.Code(err) != codes.InvalidArgument { - t.Fatalf("code = %v, want InvalidArgument (err=%v)", status.Code(err), err) - } - if accessor.gotName != "" { - t.Errorf("Secret Manager was asked for %q; a bad URI must not be dialed", accessor.gotName) - } -} - -// Only projects granted to the actor's atespace resolve, and a refused request -// never reaches Secret Manager. -func TestFetchSecretAuthorization(t *testing.T) { - const name = "projects/proj-123/secrets/example-api/versions/1" - const uri = "ate-secret://secretmanager.googleapis.com/" + name - authz, err := newProjectAuthorizer(projectPolicyFile{Policies: []atespaceProjectPolicy{ - {Atespace: "team-a", AllowedProjects: []string{"proj-123"}}, - {Atespace: "team-b", AllowedProjects: []string{"proj-456"}}, - }}) - if err != nil { - t.Fatalf("newProjectAuthorizer: %v", err) - } - - for _, tc := range []struct { - name string - authz *ProjectAuthorizer - actor string - wantCode codes.Code - }{ - {name: "project granted", authz: authz, actor: "spiffe://substrate-actor.local/actor/team-a/agent-1"}, - {name: "another project granted", authz: authz, actor: "spiffe://substrate-actor.local/actor/team-b/agent-1", wantCode: codes.PermissionDenied}, - {name: "atespace absent from the policy", authz: authz, actor: "spiffe://substrate-actor.local/actor/team-c/agent-1", wantCode: codes.PermissionDenied}, - {name: "no actor", authz: authz, actor: "", wantCode: codes.PermissionDenied}, - {name: "not an actor SPIFFE ID", authz: authz, actor: "spiffe://cluster.local/ns/team-a/sa/default", wantCode: codes.PermissionDenied}, - {name: "no policy", authz: nil, actor: "spiffe://substrate-actor.local/actor/team-a/agent-1", wantCode: codes.PermissionDenied}, - } { - t.Run(tc.name, func(t *testing.T) { - accessor := &fakeAccessor{byName: map[string][]byte{name: []byte("s3cr3t")}} - resp, err := NewServer(only(accessor), tc.authz, time.Second).FetchSecret(context.Background(), &credprovider.FetchSecretRequest{Uri: uri, ActorSpiffeId: tc.actor}) - if status.Code(err) != tc.wantCode { - t.Fatalf("code = %v, want %v (err=%v)", status.Code(err), tc.wantCode, err) - } - if tc.wantCode == codes.OK { - if got := string(resp.GetOpaqueBytes()); got != "s3cr3t" { - t.Errorf("FetchSecret = %q, want s3cr3t", got) - } - return - } - if accessor.gotName != "" { - t.Errorf("Secret Manager was asked for %q; a refused request must not reach it", accessor.gotName) - } - }) - } -} - -// A Secret Manager call that outlasts the fetch timeout fails with -// DeadlineExceeded, which the gateway retries. -func TestFetchSecretTimesOut(t *testing.T) { - const timeout = 50 * time.Millisecond - accessor := &fakeAccessor{block: true} - start := time.Now() - _, err := NewServer(only(accessor), testAuthorizer(t), timeout).FetchSecret(context.Background(), &credprovider.FetchSecretRequest{ - Uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/secrets/example-api/versions/1", - ActorSpiffeId: testActor, - }) - if status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("code = %v, want DeadlineExceeded (err=%v)", status.Code(err), err) - } - if elapsed := time.Since(start); elapsed < timeout || elapsed > 10*timeout { - t.Errorf("FetchSecret returned after %v, want about %v", elapsed, timeout) - } -} - -// Each secret is read through its location's client, by its full resource name. -func TestFetchSecretRoutesByLocation(t *testing.T) { - const global = "projects/proj-123/secrets/example-api/versions/1" - const regional = "projects/proj-123/locations/us-central1/secrets/example-api/versions/1" - globalClient := &fakeAccessor{byName: map[string][]byte{global: []byte("from-global")}} - regionalClient := &fakeAccessor{byName: map[string][]byte{regional: []byte("from-us-central1")}} - clients := func(location string) (SecretAccessor, error) { - switch location { - case "": - return globalClient, nil - case "us-central1": - return regionalClient, nil - } - return nil, fmt.Errorf("cannot create a client for %q", location) - } - srv := NewServer(clients, testAuthorizer(t), time.Second) - - for _, tc := range []struct { - name string - uri string - want string - wantCode codes.Code - }{ - {name: "global", uri: "ate-secret://secretmanager.googleapis.com/" + global, want: "from-global"}, - {name: "regional", uri: "ate-secret://secretmanager.googleapis.com/" + regional, want: "from-us-central1"}, - // A region whose client cannot be created may work on a retry. - {name: "no client for the region", uri: "ate-secret://secretmanager.googleapis.com/projects/proj-123/locations/europe-west4/secrets/example-api/versions/1", wantCode: codes.Unavailable}, - } { - t.Run(tc.name, func(t *testing.T) { - resp, err := srv.FetchSecret(context.Background(), &credprovider.FetchSecretRequest{Uri: tc.uri, ActorSpiffeId: testActor}) - if status.Code(err) != tc.wantCode { - t.Fatalf("code = %v, want %v (err=%v)", status.Code(err), tc.wantCode, err) - } - if got := string(resp.GetOpaqueBytes()); got != tc.want { - t.Errorf("FetchSecret = %q, want %q", got, tc.want) - } - }) - } - if globalClient.gotName != global || regionalClient.gotName != regional { - t.Errorf("global client read %q and regional client %q, want %q and %q", globalClient.gotName, regionalClient.gotName, global, regional) - } -} From a8ad91b4b37d468f212d071842b0e25ed19e1050 Mon Sep 17 00:00:00 2001 From: Scott Funkenhauser <85249587+sfunkenhauser@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:23:34 +0000 Subject: [PATCH 30/53] Add full name of sfunkenhauser to MAINTAINERS.md (#3384) Signed-off-by: Scott Funkenhauser --- MAINTAINERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 93767601fe..667cfa2e22 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -20,7 +20,7 @@ The current maintainers of the Agent Substrate project are listed below. | Maya Wang | mayawang | Google | | Michelle Au | msau42 | Google | | Jaana Dogan | rakyll | Google | -| | sfunkenhauser | Google | +| Scott Funkenhauser | sfunkenhauser | Google | | Tim Hockin | thockin | Google | | Zoe Zhao | zoez7 | Google | From ed3f2e4f3c5e49cc883f3a284667811421e2d4c7 Mon Sep 17 00:00:00 2001 From: Luiz Oliveira Date: Wed, 28 Oct 2026 15:41:19 +0000 Subject: [PATCH 31/53] UpdateActor rejects Actors marked for deletion (#3341) #3333 Signed-off-by: Luiz Oliveira --- cmd/ateapi/internal/controlapi/actor.go | 3 ++ .../controlapi/functionaltest/actor_test.go | 28 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index 0e56a6ae70..0977736ba3 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -264,6 +264,9 @@ func (s *RPCService) UpdateActor(ctx context.Context, req *ateapi.UpdateActorReq defer lease.Close() storedActor, err := s.impl.UpdateActor(ctx, actorRef, store.PreconditionFrom(inActor), func(toUpdate *ateapi.Actor) error { + if toUpdate.GetMetadata().GetDeleteTime() != nil { + return apierror.FailedPrecondition("Actor %s is being deleted", actorRef) + } // Status and Metadata are server-owned fields. status, metadata := toUpdate.GetStatus(), toUpdate.GetMetadata() // Whole-object replace: clear first, so a field the client left unset is diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 0934da4e22..7f169fb8cf 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -1058,6 +1058,34 @@ func TestUpdateActor_NotFound(t *testing.T) { assertGrpcError(t, err, codes.NotFound, "actor test-atespace/does-not-exist not found") } +func TestUpdateActor_FailsIfDeleting(t *testing.T) { + ns := namespaceForTest("ns-update-actor-deleting") + tc := setupTest(t, ns) + defer tc.cleanup() + createTemplate(t, tc, ns) + + ctx := context.Background() + actorRef := resources.ActorRef{Atespace: testAtespace, Name: "id1"} + if _, err := tc.persistence.CreateActor(ctx, &ateapi.Actor{ + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + Status: &ateapi.ActorStatus{State: ateapi.ActorState_ACTOR_STATE_DELETING}, + ActorTemplate: &ateapi.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }); err != nil { + t.Fatalf("CreateActor: %v", err) + } + marked, err := tc.persistence.MarkActorForDeletion(ctx, actorRef, store.DeletePreconditions{}, nil) + if err != nil { + t.Fatalf("MarkActorForDeletion: %v", err) + } + + _, err = tc.client.UpdateActor(ctx, &ateapi.UpdateActorRequest{Actor: &ateapi.Actor{ + Metadata: marked.GetMetadata(), + ActorTemplate: marked.GetActorTemplate(), + WorkerSelector: &ateapi.Selector{MatchLabels: map[string]string{"tier": "paid"}}, + }}) + assertGrpcError(t, err, codes.FailedPrecondition, "Actor test-atespace/id1 is being deleted") +} + func TestUpdateActor_StampsFullSpanIdentity(t *testing.T) { ns := namespaceForTest("ns-span-update") tc := setupTest(t, ns) From 534fe7b068cda0fb77ba7d57ef8883ca89bd4e78 Mon Sep 17 00:00:00 2001 From: Greg Castle Date: Wed, 28 Oct 2026 17:00:11 +0000 Subject: [PATCH 32/53] docs: add extensibility guide for gateways (#3317) Add a dedicated guide describing how to augment or replace Substrate's ingress and egress gateways via ext_proc callouts or bring-your-own gateways. Replaces the outdated extensibility section in `docs/security.md` and links tracking issues #3258 and #3259. AI use: some help with drafting. I reviewed and iterated. - [x] Tests pass: no code - [x] Appropriate changes to documentation are included in the PR --------- Signed-off-by: Greg Castle --- README.md | 1 + docs/extensibility.md | 26 ++++++++++++++++++++++++++ docs/security.md | 5 +---- 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 docs/extensibility.md diff --git a/README.md b/README.md index acdcecaf84..253d7c47b0 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * [Glossary](docs/glossary.md): Core terms (Actor, Atespace, ActorTemplate, WorkerPool, Worker, ate-api-server, atenet, atelet, ateworker) and how they relate. * [Integration Repositories](docs/integration-repos.md): Where integrations live, how their repositories are named, and how fixes flow back to core. * [Observability Guide](docs/observability.md): Guide to actor logging, metrics, and distributed tracing. +* [Extending Substrate](docs/extensibility.md): How to augment or replace the built-in ingress and egress gateways. * [Authentication Guide](docs/authentication.md): Configure trusted JWT providers and human credentials. * [Egress Traffic](docs/egress-traffic.md): Which protocols an Actor may reach the outside world with and which are blocked. * [Enabling man-in-the-middle (MITM) interception for Actor Egress policy](docs/egress-trust-bundle.md): Egress policies such as header injection depend on MITM interception of Actor traffic. This guide explains how an Actor should be configured to enable interception. diff --git a/docs/extensibility.md b/docs/extensibility.md new file mode 100644 index 0000000000..ddf5b89492 --- /dev/null +++ b/docs/extensibility.md @@ -0,0 +1,26 @@ +# Extending Substrate + +Substrate ships with built-in sandboxing, actor identity, and ingress and egress gateways. Security vendors and platform teams can plug their own capabilities into the gateways, either to **augment** the built-in controls or to **replace** them. + +| Path | Ingress | Egress | +| --- | --- | --- | +| **Augment:** add your `ext_proc` callout to Substrate's gateway | Planned ([#3259](https://github.com/agent-substrate/substrate/issues/3259)) | Available (experimental) | +| **Replace:** bring your own gateway | Planned ([#3258](https://github.com/agent-substrate/substrate/issues/3258)) | Partial | + +## Augment: add an ext_proc callout + +You can add your own [`ext_proc`](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/ext_proc_filter) service. Substrate gateways call it on every request. + +- **Egress:** set `--experimental-additional-egress-extproc-service` (see [Operator install](operator-install.md#settings)). This may be converted to an API in the future. Your service runs after Substrate has authenticated the actor and applied its `EgressPolicy`. It currently receives the request headers and the actor's identity, and it can deny the request or add headers; this can be changed if necessary. Today it sees headers only, not bodies. +- **Ingress:** planned. + +## Replace: bring your own gateway + +- **Egress:** you can point actors at your own policy enforcement point (PEP) instead of `atenet-egress`. Your PEP must meet the [Network Egress Contract](network-egress.md): it must verify the actor's certificate and reject any traffic it can't attribute to a running actor. Honoring Substrate's `EgressPolicy` is encouraged but not required. The installer doesn't have a bring-your-own mode yet. +- **Ingress:** planned. `atenet-router` does more than filter traffic: it wakes suspended actors and routes requests to their workers. A replacement router will need to call Substrate, either through the Substrate ext_proc or via the Substrate API, for those steps. + +If you replace a gateway, the authentication and authorization it enforced become your responsibility. + +## Get help with integrations + +We want to help you integrate, and we're discussing other ways to extend Substrate beyond gateways. Talk to us in [#substrate-dev](https://cloud-native.slack.com/archives/C0B6M3E2J3D) on CNCF Slack, [open a GitHub issue](https://github.com/agent-substrate/substrate/issues/new), or join the weekly [community meeting](../README.md#community). diff --git a/docs/security.md b/docs/security.md index 675d511ab8..3bb01e149f 100644 --- a/docs/security.md +++ b/docs/security.md @@ -20,10 +20,7 @@ We expect the application layer creating actors using substrate to be responsibl ## Security Extensibility -More advanced security can be integrated using the ingress and egress extension points. This is where advanced prompt injection scanning, dynamic LLM-driven authorization decisions, data leak prevention (DLP), or cloud IAM policy enforcement can be added: - -- **Egress Extensibility**: The outbound path is a fully pluggable Policy Enforcement Point (PEP). Operators can either extend the existing egress proxy using Envoy's `ext_proc` callout mechanism to invoke external inspection and authorization services, or completely replace the egress gateway (`atenet-egress`) with an external solution. -- **Ingress Extensibility**: Ingress similarly supports `ext_proc` callouts for external authentication, authorization, and payload inspection. However, unlike egress, the ingress gateway (`atenet-router`) cannot be completely replaced from the outside: Substrate's core orchestration relies on ingress routing hooks to manage actor state, resolve worker assignments, and trigger on-demand cold starts or resume operations from snapshots to wake up agents. +Substrate provides extension points for operators and vendors to augment or replace the built-in ingress and egress gateways. See [Extending Substrate](extensibility.md). ## Core Security Principles From 38874cedfb1d46026aceb91c74fb78c09b550925 Mon Sep 17 00:00:00 2001 From: Luiz Oliveira Date: Wed, 28 Oct 2026 17:14:10 +0000 Subject: [PATCH 33/53] CreateActor fails if [golden] tag is marked for deletion (#3382) #3333 Signed-off-by: Luiz Oliveira --- cmd/ateapi/internal/controlapi/actor.go | 7 ++- .../controlapi/functionaltest/actor_test.go | 54 +++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index 0977736ba3..bd092788e0 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -191,8 +191,11 @@ func (s *ServiceImpl) resolveTagSource(ctx context.Context, actorAtespace string default: return nil, apierror.FailedPrecondition("source Tag has an invalid scope") } - // TODO: Refuse a tag with delete_time set. DeleteTag may already have - // collected its snapshot. + // DeleteTag may already have collected the snapshot of a tag with + // delete_time set. + if tag.GetMetadata().GetDeleteTime() != nil { + return nil, apierror.FailedPrecondition("source Tag %s/%s is being deleted", tagRef.GetAtespace(), tagRef.GetName()) + } // A tag might have an empty Snapshot URI if the tag creation failed or is ongoing. if tagDurableSnapshotURI(tag, ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED) == "" { return nil, apierror.FailedPrecondition("source Tag is still being created or failed creation") diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 7f169fb8cf..bd49b56f88 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -488,6 +488,60 @@ func TestCreateActor_PendingTag(t *testing.T) { } } +// TestCreateActor_FromDeletingTag verifies an Actor cannot be created from a tag +// with delete_time set, since DeleteTag may already have collected its snapshot. +func TestCreateActor_FromDeletingTag(t *testing.T) { + ns := namespaceForTest("ns-deleting-tag") + tc := setupTest(t, ns) + defer tc.cleanup() + ctx := context.Background() + tmpl := createTemplate(t, tc, ns) + + tag := seedTag(t, tc, "deleting-source", "deleting", func(tag *ateapi.Tag) { + tag.Status.ActorTemplateUid = tmpl.GetMetadata().GetUid() + }) + if _, err := tc.persistence.MarkTagForDeletion(ctx, resources.TagRefFromTag(tag), store.DeletePreconditions{}, nil); err != nil { + t.Fatalf("MarkTagForDeletion: %v", err) + } + + _, err := tc.client.CreateActor(ctx, &ateapi.CreateActorRequest{Actor: &ateapi.Actor{ + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "clone"}, + ActorTemplate: &ateapi.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + SourceTag: resources.TagRefFromTag(tag).ToObjectRef(), + }}) + assertGrpcError(t, err, codes.FailedPrecondition, "source Tag "+testAtespace+"/deleting is being deleted") +} + +// TestCreateActor_FromDeletingGoldenTag verifies an Actor created without a source +// tag is refused while its template's golden tag has delete_time set. +func TestCreateActor_FromDeletingGoldenTag(t *testing.T) { + ns := namespaceForTest("ns-deleting-golden-tag") + tc := setupTest(t, ns) + defer tc.cleanup() + ctx := context.Background() + tmpl := createTemplate(t, tc, ns) + + golden := seedTag(t, tc, "golden-source", "golden", func(tag *ateapi.Tag) { + tag.Status.ActorTemplateUid = tmpl.GetMetadata().GetUid() + }) + goldenRef := resources.TagRefFromTag(golden) + if _, err := tc.persistence.UpdateActorTemplate(ctx, resources.ActorTemplateRefFromActorTemplate(tmpl), store.PreconditionFrom(tmpl), func(toUpdate *ateapi.ActorTemplate) error { + toUpdate.Status = &ateapi.ActorTemplateStatus{GoldenSnapshotStatus: &ateapi.GoldenSnapshotStatus{GoldenTag: goldenRef.ToObjectRef()}} + return nil + }); err != nil { + t.Fatalf("recording the golden tag: %v", err) + } + if _, err := tc.persistence.MarkTagForDeletion(ctx, goldenRef, store.DeletePreconditions{}, nil); err != nil { + t.Fatalf("MarkTagForDeletion: %v", err) + } + + _, err := tc.client.CreateActor(ctx, &ateapi.CreateActorRequest{Actor: &ateapi.Actor{ + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "actor"}, + ActorTemplate: &ateapi.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}) + assertGrpcError(t, err, codes.FailedPrecondition, "source Tag "+testAtespace+"/golden is being deleted") +} + // TestGetActor_Found tests that an existing actor can be retrieved. func TestGetActor_Found(t *testing.T) { ns := namespaceForTest("ns-get-found") From 55103007a3948df30fca8ddc80d444422c18e51f Mon Sep 17 00:00:00 2001 From: Yufan Su Date: Wed, 28 Oct 2026 17:32:25 +0000 Subject: [PATCH 34/53] doc: add details on how a DNS denial from egress policy looks inside the actor (#3363) Adds a note to `docs/egress-traffic.md` with the errors to expect and the command to check the actor's egress policy. Part of #3228. - [ ] Tests pass - [x] Docs updated Signed-off-by: Yufan Su --- docs/egress-traffic.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/egress-traffic.md b/docs/egress-traffic.md index 224448ccbb..fb72475870 100644 --- a/docs/egress-traffic.md +++ b/docs/egress-traffic.md @@ -10,6 +10,13 @@ policy, or for a hostname not allowed by any rule, are answered `NXDOMAIN`. A policy created, updated, or deleted while the actor runs applies to its DNS within about 5 seconds. +> [!NOTE] +> The actor sees a denied name as an ordinary resolver failure, such as +> `wget: bad address 'api.github.com'` or `curl: (6) Could not resolve host`. +> Nothing logs the denied name. If an external name does not resolve, check that +> the actor's egress policy has a rule for it with +> `kubectl ate get egress-policy -a -o yaml`. + Other UDP and other traffic is filtered by nftables and never reaches the gateway. This applies to an actor's own telemetry too: an OpenTelemetry SDK inside the actor From 34491cc4d0bcaba9c79107b75fa4f28ef2a9827f Mon Sep 17 00:00:00 2001 From: Adrian Ludwin Date: Wed, 28 Oct 2026 17:37:35 +0000 Subject: [PATCH 35/53] Add documentation for SLOs and alerting (#2476) This document covers proposed SLIs, SLOs, and non-SLO alerts to monitor essential surfaces in Substrate. Some lower-priority surfaces are still tbd but this is likely good enough for v1.0, as sophisticated operators can likely self-unblock given existing observability. I tested the configurations shown here on GKE 1.35 using GMP, and there are some idiosyncrasies in the code samples as a result. I've tried to call these all out in comments to make it clear how operators can clean them up, either now (e.g. if they're using Prometheus directly and not GMP) or in the future (when the idiosyncrasies are fixed). Fixes #574 - [ n/a ] Tests pass - [ x ] Appropriate changes to documentation are included in the PR Signed-off-by: Adrian Ludwin --- docs/examples/gmp-alerting-rules.yaml | 357 ++++++++++++++++++++++++ docs/observability.md | 4 + docs/slos-and-alerts.md | 382 ++++++++++++++++++++++++++ 3 files changed, 743 insertions(+) create mode 100644 docs/examples/gmp-alerting-rules.yaml create mode 100644 docs/slos-and-alerts.md diff --git a/docs/examples/gmp-alerting-rules.yaml b/docs/examples/gmp-alerting-rules.yaml new file mode 100644 index 0000000000..2519c5a8a7 --- /dev/null +++ b/docs/examples/gmp-alerting-rules.yaml @@ -0,0 +1,357 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# See ../slos-and-alerts.md for context on this file. +# +# This file is a Custom Resource for Google Cloud Managed Service for +# Prometheus ("GMP"), but everything inside of the "spec" is identical to +# Prometheus' `rule_group` array, so it can be used by any compliant Prometheus +# installation, with the exceptions listed below. See +# https://docs.cloud.google.com/stackdriver/docs/managed-prometheus/rules-managed +# for more information. +# +# The known differences in this file to support GMP (vs regular Prometheus) are: +# +# - **Dotted OTel label names:** GMP (up to at least GKE 1.37) rejects +# dotted OTel label names in `sum by (...)` / `max by (...)` clauses, so the +# rules copy labels like `rpc.method` to `rpc_method` via `label_replace` +# first (see comments below). +# - **Workload name selectors:** GMP automatically attaches +# `top_level_controller_name` from the Kubernetes Deployment name. +# Outside GMP, replace this selector with the label your Prometheus setup +# uses to identify the API server (e.g. `job="ateapi"`, +# `app="ate-api-server"`, or `"service.name"="ateapi"`; see comment at first +# use below). +# - **Metric unit suffixes and naming:** GMP preserves OTel dotted instrument names +# and omits unit suffixes on histograms; Standard Prometheus exporters append +# the unit suffix (e.g. `_seconds_count` / `.seconds_count`) and, unless +# UTF-8 naming is enabled with NoTranslation (which should be the default after #2343), +# translate dots to underscores. You may need to adjust the metric names as a result; +# see [metric naming](../dev/best-practices/metrics.md) for more information. +# +# To test this file, you can set up a very basic "alerting" configuration as +# follows: +# +# 1. Create an HTTP-to-stdout pod (use this pod image at your own risk): +# +# kubectl run alert-logger --image=mendhak/http-https-echo:42 --port=8080 -n gmp-public +# kubectl expose pod alert-logger --port=8080 --target-port=8080 -n gmp-public +# +# 2. Create an alertmanager configuration: +# +# global: +# resolve_timeout: 5m +# route: +# receiver: 'test-webhook' +# receivers: +# - name: 'test-webhook' +# webhook_configs: +# - url: 'http://alert-logger.gmp-public.svc.cluster.local:8080/' +# send_resolved: true +# +# 3. Point GMP to the alertmanager file (if not using GMP, see Prom documentation): +# +# kubectl create secret generic alertmanager -n gmp-public --from-file=alertmanager.yaml +# +# 4. Apply this file, then watch the logs coming off of the alert-logger pod to +# see when alerts are fired or resolved. +apiVersion: monitoring.googleapis.com/v1 +kind: ClusterRules +metadata: + name: substrate-slos-and-alerts +spec: + groups: + # ============================================================================ + # ResumeActor Capacity Availability (ResourceExhausted vs Success) + # See section A.1 in ../slos-and-alerts.md. + # Target: 99% capacity availability over 30d (error budget = 0.01) + # For strategy, see https://sre.google/workbook/alerting-on-slos/ + # + # **TODO:** Find a way to distinguish between retryable and final failures, e.g. + # by passing in a "retryable" parameter in ResumeActor. See + # [#2650](https://github.com/agent-substrate/substrate/issues/2650). + # ============================================================================ + - name: ate_resume_actor_capacity_slo_recording_rules + rules: + # 1. Base 5m event rates (the only rules that scan the raw OTel metric) + # + # NB: top_level_controller_name="ate-api-server" is a GMP-specific label + # added by Google Cloud Monitoring from the Kubernetes Deployment name. + # Outside GMP, replace it throughout this file with the label your + # Prometheus setup attaches to the API server, such as: + # - job="ateapi" (when exporting via the OTel Collector Prometheus exporter, + # which maps the resource attribute service.name="ateapi" to job) + # - app="ate-api-server" (when scraping pods directly with pod labelmapping, + # as in manifests/ate-install/kind/prometheus.yaml) + # - "service.name"="ateapi" (when promoting OTel resource attributes) + - record: aterules:actor_resume_no_capacity:rate5m + expr: | + sum(rate({"ate.actor.lifecycle.operation.duration_count", top_level_controller_name="ate-api-server", "ate.actor.operation.name"="resume", "error.type"="ResourceExhausted"}[5m])) + or vector(0) + - record: aterules:actor_resume_valid:rate5m + # NB: this doesn't filter out errors other than ResourceExhausted; it + # looks at _all_ calls. That can probably be tightened up, but combined + # with the RPC SLOs this should be good enough. + expr: | + sum(rate({"ate.actor.lifecycle.operation.duration_count", top_level_controller_name="ate-api-server", "ate.actor.operation.name"="resume"}[5m])) + or vector(0) + # 2. Pre-computed multi-window error ratios (rolled up from the 5m rates) + - record: aterules:actor_resume_no_capacity:ratio_rate5m + expr: | + aterules:actor_resume_no_capacity:rate5m + / + aterules:actor_resume_valid:rate5m + - record: aterules:actor_resume_no_capacity:ratio_rate30m + expr: | + avg_over_time(aterules:actor_resume_no_capacity:rate5m[30m]) + / + avg_over_time(aterules:actor_resume_valid:rate5m[30m]) + - record: aterules:actor_resume_no_capacity:ratio_rate1h + expr: | + avg_over_time(aterules:actor_resume_no_capacity:rate5m[1h]) + / + avg_over_time(aterules:actor_resume_valid:rate5m[1h]) + - record: aterules:actor_resume_no_capacity:ratio_rate6h + expr: | + avg_over_time(aterules:actor_resume_no_capacity:rate5m[6h]) + / + avg_over_time(aterules:actor_resume_valid:rate5m[6h]) + - record: aterules:actor_resume_no_capacity:ratio_rate3d + expr: | + avg_over_time(aterules:actor_resume_no_capacity:rate5m[3d]) + / + avg_over_time(aterules:actor_resume_valid:rate5m[3d]) + - name: ate_resume_actor_capacity_slo_alerts + rules: + # Page on fast burn (14.4x over 1h/5m) OR slow burn (6x over 6h/30m) + - alert: AteResumeActorCapacityBurnPage + expr: | + ( + aterules:actor_resume_no_capacity:ratio_rate1h > (14.4 * 0.01) + and + aterules:actor_resume_no_capacity:ratio_rate5m > (14.4 * 0.01) + ) + or + ( + aterules:actor_resume_no_capacity:ratio_rate6h > (6 * 0.01) + and + aterules:actor_resume_no_capacity:ratio_rate30m > (6 * 0.01) + ) + labels: + severity: page + slo: ate-resume-actor-capacity + annotations: + summary: "High ResumeActor out-of-capacity error budget burn rate" + description: "ResumeActor ResourceExhausted failures are burning the 30d 99% capacity budget at a paging rate (current={{ $value | humanizePercentage }})." + # Ticket on sustained burn (1x over 3d/6h) + - alert: AteResumeActorCapacityBurnTicket + expr: | + aterules:actor_resume_no_capacity:ratio_rate3d > (1 * 0.01) + and + aterules:actor_resume_no_capacity:ratio_rate6h > (1 * 0.01) + labels: + severity: ticket + slo: ate-resume-actor-capacity + annotations: + summary: "Sustained ResumeActor out-of-capacity error budget burn" + description: "ResumeActor ResourceExhausted failure rate has exceeded the 1x burn rate threshold over 3d and 6h." + # ============================================================================ + # ateapi Per-Method RPC Server-Side Error Rate (5xx-equivalent gRPC codes) + # See section A.2 in ../slos-and-alerts.md. + # Target: 99.9% availability over 30d per RPC method (error budget = 0.001) + # For strategy, see https://sre.google/workbook/alerting-on-slos/ + # + # NB: For low-traffic methods (e.g. CreateActorTemplate), this could be much + # too sensitive. If this applies to you, consider some of the options from + # the SRE workbook: combine multiple methods, exclude those methods, or + # generate artificial traffic. + # + # **Note:** these sample rules only cover availability, not latency, but the + # availability SLOs can easily be adapted into multiple tiers of latency SLOs + # as needed. + # ============================================================================ + - name: ateapi_rpc_availability_slo_recording_rules + rules: + # 1. Base 5m event rates per rpc.method (the only rules that scan the raw OTel metric) + - record: aterules:api_rpc_calls:rate5m + # In Google Managed Prometheus, at least up to and including GKE 1.37, labels in a "sum by(xxx)" + # expression can only include traditional names (e.g. rpc_method) and not Otel-style names + # (e.g. rpc.method). This is due to github.com/GoogleCloudPlatform/prometheus-engine depending + # on an older version of the Prometheus parser (v2.53.5); this dependency was updated on + # 23-09-2026 (5c449d70c4) to v3.13.0, which fixes this bug, but it may take some time for this + # to be released in GKE. + # + # If you are using an updated version of GKE, or are not using GMP, you can remove the + # label_replace functions (in this rule and the next), and replace all occurrences of rpc_method + # with rpc.method in the rest of this file. + expr: | + sum by (rpc_method) ( + label_replace( + rate({"rpc.server.call.duration_count", top_level_controller_name="ate-api-server"}[5m]), + "rpc_method", "$1", "rpc.method", "(.*)" + ) + ) + + - record: aterules:api_rpc_server_errors:rate5m + expr: | + sum by (rpc_method) ( + label_replace( + rate({"rpc.server.call.duration_count", top_level_controller_name="ate-api-server", "rpc.response.status_code"=~"INTERNAL|UNAVAILABLE|DATA_LOSS|UNKNOWN|DEADLINE_EXCEEDED|UNIMPLEMENTED"}[5m]), + "rpc_method", "$1", "rpc.method", "(.*)" + ) + ) + or on (rpc_method) + sum by (rpc_method) (0 * aterules:api_rpc_calls:rate5m) + + # 2. Pre-computed multi-window error ratios per rpc.method (rolled up from the 5m rates) + - record: aterules:api_rpc_server_errors:ratio_rate5m + expr: | + aterules:api_rpc_server_errors:rate5m + / + aterules:api_rpc_calls:rate5m + - record: aterules:api_rpc_server_errors:ratio_rate30m + expr: | + avg_over_time(aterules:api_rpc_server_errors:rate5m[30m]) + / + avg_over_time(aterules:api_rpc_calls:rate5m[30m]) + - record: aterules:api_rpc_server_errors:ratio_rate1h + expr: | + avg_over_time(aterules:api_rpc_server_errors:rate5m[1h]) + / + avg_over_time(aterules:api_rpc_calls:rate5m[1h]) + - record: aterules:api_rpc_server_errors:ratio_rate6h + expr: | + avg_over_time(aterules:api_rpc_server_errors:rate5m[6h]) + / + avg_over_time(aterules:api_rpc_calls:rate5m[6h]) + - record: aterules:api_rpc_server_errors:ratio_rate3d + expr: | + avg_over_time(aterules:api_rpc_server_errors:rate5m[3d]) + / + avg_over_time(aterules:api_rpc_calls:rate5m[3d]) + - name: ateapi_rpc_availability_slo_alerts + rules: + # Page if ANY rpc.method hits fast burn (14.4x over 1h/5m) OR slow burn (6x over 6h/30m) + - alert: AteApiRpcErrorBudgetBurnPage + expr: | + ( + aterules:api_rpc_server_errors:ratio_rate1h > (14.4 * 0.001) + and + aterules:api_rpc_server_errors:ratio_rate5m > (14.4 * 0.001) + ) + or + ( + aterules:api_rpc_server_errors:ratio_rate6h > (6 * 0.001) + and + aterules:api_rpc_server_errors:ratio_rate30m > (6 * 0.001) + ) + labels: + severity: page + slo: ateapi-rpc-availability + annotations: + summary: "High server-side error budget burn rate on {{ index $labels \"rpc_method\" }}" + description: "ateapi RPC {{ index $labels \"rpc_method\" }} 5xx-equivalent gRPC error rate is burning the 30d 99.9% error budget at a paging rate (current={{ $value | humanizePercentage }})." + + # Ticket if ANY rpc.method hits sustained burn (1x over 3d/6h) + - alert: AteApiRpcErrorBudgetBurnTicket + expr: | + aterules:api_rpc_server_errors:ratio_rate3d > (1 * 0.001) + and + aterules:api_rpc_server_errors:ratio_rate6h > (1 * 0.001) + labels: + severity: ticket + slo: ateapi-rpc-availability + annotations: + summary: "Sustained server-side error budget burn on {{ index $labels \"rpc_method\" }}" + description: "ateapi RPC {{ index $labels \"rpc_method\" }} 5xx-equivalent gRPC error rate has exceeded the 1x burn rate threshold over 3d and 6h." + # ============================================================================ + # Alert on lack of convergence between desired and actual number of workers. + # + # See section B.1 in ../slos-and-alerts.md. NB: this alert fires on the + # _overall_ failure rate, of intended workers -> registered pods; that is, it + # fires whether the failure is because of the pods not being created, or the + # pods not being registered. To debug which of these it is, look at + # ate.workerpool.ready_workers which tracks the number of ready _pods_. If + # that is converging, but the overall number of workers is not, there's a + # problem with registration; otherwise, your cluster is likely out of + # capacity (though you will likely see other symptoms of this, e.g. + # unschedulable pods). + # + # The strategy here is: + # - Look for workers in all states except "unschedulable." Ignore unknown + # sandbox classes. + # - Sum across all (remaining) workload states (e.g. excluding + # "unschedulable"). Alert if this is lower than the lowest number of + # desired workers over the last five minutes. + # - Each replica of ate-api-server emits these metrics, so we need a way to + # resolve disagreements between them on the correct number of actual and + # desired workers. The most likely conflict is that a replica disappears + # (presumably, to later be replaced) and its last value is therefore "stuck," + # or that one of them is a bit slow. Therefore, we take the max value across + # all replicas for the current number of workers, and the min value for the + # desired number of workers, so that if max(current) < min(desired), we are + # *certain* that something has gone wrong. This may miss some genuine problems + # but those should be transient. + # ============================================================================ + - name: ate_workerpool_recording_rules + rules: + # See comment for label_replace, above. You can remove this and simplify + # the alerts below if you are not using GMP. + - record: aterules:workerpool_actual_workers:label_replaced + expr: | + label_replace( + label_replace( + label_replace( + label_replace( + {"ate.workerpool.workers", "ate.worker.state"!="unschedulable", "ate.sandbox.class"!="unknown"}, + "ate_sandbox_class", "$1", "ate.sandbox.class", "(.*)" + ), + "ate_worker_state", "$1", "ate.worker.state", "(.*)" + ), + "ate_workerpool_name", "$1", "ate.workerpool.name", "(.*)" + ), + "ate_workerpool_namespace", "$1", "ate.workerpool.namespace", "(.*)" + ) + - record: aterules:workerpool_desired_workers:label_replaced + expr: | + label_replace( + label_replace( + {"ate.workerpool.desired_workers"}, + "ate_workerpool_name", "$1", "ate.workerpool.name", "(.*)" + ), + "ate_workerpool_namespace", "$1", "ate.workerpool.namespace", "(.*)" + ) + + - name: ate_workerpool_alerts + rules: + - alert: SubstrateWorkerPoolCapacityNotDelivered + expr: | + sum by (ate_workerpool_namespace, ate_workerpool_name) ( + max by (ate_workerpool_namespace, ate_workerpool_name, ate_worker_state, ate_sandbox_class) ( + aterules:workerpool_actual_workers:label_replaced + ) + ) + < + min by (ate_workerpool_namespace, ate_workerpool_name) ( + min_over_time(aterules:workerpool_desired_workers:label_replaced[5m]) + ) + for: 5m + labels: + severity: warning + annotations: + summary: 'WorkerPool {{ index $labels "ate_workerpool_namespace" }}/{{ index $labels "ate_workerpool_name" }} has not reached target capacity in ateapi' + description: >- + WorkerPool {{ index $labels "ate_workerpool_namespace" }}/{{ index $labels "ate_workerpool_name" }} + currently has {{ $value }} schedulable workers registered in ateapi, + which is fewer than the fewest number of workers requested over the last five minutes. diff --git a/docs/observability.md b/docs/observability.md index ab93ba11b4..f8cffd84d3 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -428,6 +428,10 @@ To explore metrics locally: > **Roadmap Note (Actor-Level Metrics):** The control plane latency, snapshot performance, and fleet utilization instruments above are in place. What remains is actor-level correlation: telemetry that an actor emits itself carries no Substrate-injected identity yet ([#853](https://github.com/agent-substrate/substrate/issues/853), [#761](https://github.com/agent-substrate/substrate/issues/761)). +### SLOs and alerting + +Please see the [SLOs and alerting](slos-and-alerts.md) page for suggestions about how to set up SLOs, SLO-based alerts, and other alerts to ensure your deployment is working well. + --- ## 3. Tracing diff --git a/docs/slos-and-alerts.md b/docs/slos-and-alerts.md new file mode 100644 index 0000000000..761006109c --- /dev/null +++ b/docs/slos-and-alerts.md @@ -0,0 +1,382 @@ +# SLOs and alerting + +This page explains how operators can +[defend](#appendix-monitoring-ate-and-defending-slos) SLOs and monitor ATE in +production. It defines SLIs, suggests SLOs, and suggests other alerts operators +can use to ensure their ATE deployments are working correctly. + +# Background + +## Goals + +Operators need to monitor their ATE deployments for two reasons: + +* **Incident management:** Incidents include cases such as bad ATE releases, + problematic clients, or other misconfigurations of their ATE deployment that + lead to bad behavior for their users. +* **Optimizing tradeoffs:** Operators can set a "reasonable" rate of failed + actor resumptions to maintain a high density, or vice versa. There are other + cases where operators may want to trade off factors such as availability and + performance for cost. + +All the scenarios below are tagged with "ICT" if they could represent incidents, +or "OPT" if they help optimize the deployment. Of course, some scenarios can +represent both. + +## Surfaces + +Each primary surface of ATE needs alerting to meet the goals above. These +surfaces are: + +* **ateapi (API server):** Start and manage actors, snapshots, etc. +* **K8s APIs and operations:** Create the ATE installation itself and ensure + it's healthy; create and manage worker pools. +* **Actor execution:** As actors execute, they must be able to use local \*nix + mechanisms like process exits and signals (e.g. OOMs), exposing probers, and + producing stdout/stderr. We include mounting access to storage under this + category. +* **Actor network requests:** Actors can accept incoming network requests, even + if the actor is currently suspended, and can make outbound network requests + through the ATE-managed gateway. + +The following surfaces are *not* considered to be primary, at least for now, +because their behavior is easily observable through a different (primary) +surface. Observability of these surfaces is important for diagnosing problems, +but is not required for alerting: + +* **Snapshot storage:** The time spent storing or retrieving an actor's snapshot + is visible via the API server, but see also section B.4 below for some + subtleties in how the times should be interpreted. +* **PostgreSQL or other backends:** Similarly, any problems here should be + visible in the API server or other surfaces. + +Finally, the following are out of scope: + +* **Sandbox operation:** ATE is not responsible for the correct operation of the + underlying sandbox technology (e.g. gVisor, microVMs); that is, ATE could work + correctly even though the sandboxes are running slowly. These components are + pulled directly from upstream; see the documentation of those projects for + observability options and recommended monitoring. + +# High-level approach + +This document defines three types of signals and associated alerting. + +## SLIs, SLOs, and SLO-based alerts + +These are traditional alerts that cover good and bad events, and have error +budgets. SLIs and SLOs are expressed using PromQL rules, following the [SRE +workbook standard](https://sre.google/workbook/alerting-on-slos/). This document +uses OTel-style metric names (e.g. UTF-8 metrics). Operators are encouraged to +review and customize these alerts, including converting to traditional +Prometheus syntax if needed. + +While ATE developers cannot *defend* SLOs, we will *demonstrate* that the P0 and +P1 SLOs can be met under reasonable conditions. (**details tbd**). + +## Non-SLI metrics and custom alerts + +These are conditions that can be described as metrics, but not good-bad events +and don't have an error budget. Examples include K8s-style "not-at-intent" +problems, such as the intended and actual number of workers not converging after +a period of time. It's not an error condition for the two to be out of sync +(e.g. if the number of intended workers is constantly changing), but it is an +error if they are not constantly converging. + +As with SLOs, recommended alerts are expressed as Prometheus configs. + +## Error logs + +Many other errors are not worth having dedicated metrics; these often include +failures that will *eventually* cause another recommended SLO or alert to fail. +While we *could* introduce a one-off metric for every possible failure +condition, operators would likely not update their alerting configs and it seems +like unnecessary work to keep them in sync. + +The problem with using logging for alerting (as opposed to debugging or +observability) is that it is hard to enforce that error messages are only +produced when there are genuine, non-retryable, actionable conditions that an +operator needs to investigate. While we will attempt to reduce the number of +unactionable errors that ATE components emit, it's unlikely we'll get to (or +stay at) zero, especially considering errors that could come from libraries. + +This may seem resolvable if we introduce a new general-purpose "failure" metric +that can be set by many parts of the code, but unfortunately the same problem +would eventually apply: some authors might overuse this metric, and others might +ignore it. This is a fundamental problem associated with providing a complex +piece of infrastructure with a huge number of points of failures. + +As a result, operators will have two options: + +* They could configure alerts whenever the error rate breaches some threshold, + and accept that occasionally this will lead to false alerts. +* They could manually monitor error messages at high-risk times, e.g. while + upgrading Substrate. + +Finally, in as many cases as possible in the scenarios below, we've described +what happens if error messages are missed. In many cases, this will cause +another SLO or signal to move, but potentially much later when the situation is +already affecting users and/or is harder to resolve. There are a few cases where +this isn't true (e.g. losing stdout from actors), but these are generally +lower-severity problems. + +# Initial set of monitoring scenarios + +In this section, we discuss the general approach for each surface listed above, +as well as specific CUJs within that surface and how those CUJs will be +monitored and alerted. + +Ultimately, the owners of each ATE surface will be responsible for implementing +SLOs, non-SLO alerts, and error logs as described above, and this document will +become incomplete and/or out of date. The purpose of this section is to list the +current (as of late 2026), most critical alerting requirements that we'll +implement up-front, both to kickstart the observability, and also to illustrate +how the approach described above will actually work in practice. + +## A: API server + +All metrics related to `ate-api-server` are reported per-RPC, including +pass/fail results and latency. + +### \[ICT, OPT\] A.1: Actors cannot be executed at the expected success rate + +There are legitimate reasons that actors might not be able to be run; most +importantly, the operator may not choose to size the worker pool for absolute +peak capacity. Therefore, the failure rate of `ResumeActor` due to exhaustion is +a critical SLI, with one alert for P1 (e.g. pool is not large enough) and a +lower threshold for P0 (e.g. most or all workers are broken). (For failures due +to causes other than exhaustion, see section A.2). + +Note that this configuration uses `ate.actor.lifecycle.operation`, not +`rpc.server.call`, since the former correctly (in this case) excludes "resume" +operations where the actor was already running, while the latter does not. + +One current weakness of this alert is that it greatly overcounts failures that +occur during request parking, since request parking can retry about 20 times. +(100ms initial delay, 1.1x multiplier, 5s budget). Right now, there's no way to +tell the difference between "retryable" failures and "final" failures. + +**Example:** In the [sample rules](examples/gmp-alerting-rules.yaml), see +`ate_resume_actor_capacity_slo_recording_rules` and +`ate_resume_actor_capacity_slo_alerts`. + + +### \[ICT\] A.2: The API server or its dependencies are broken or degraded + +Most operations should be insensitive to the state of the system, and we should +maintain very high (e.g. \>99.9%) success rate. These are fairly +straightforward. + +These SLIs/SLOs only track the equivalent of 5xx codes. Notably, resource +exhaustion is excluded; see section A.1 for that. + +**Example:** In the [sample rules](examples/gmp-alerting-rules.yaml), see +`ateapi_rpc_availability_slo_recording_rules` and +`ateapi_rpc_availability_slo_alerts`. Note that these may be too sensitive for +low-traffic methods (e.g. `CreateActorTemplate`); see the file for possible +approaches to resolve this. + +_NB: the sample rules only cover availability, not latency, but the availability +SLOs can easily be adapted into multiple tiers of latency SLOs as needed._ + + +## B: K8s APIs and asynchronous operations + +Substrate introduces the following CRDs: + +* `WorkerPool`: declares and manages workers. Inherently K8s-centric since it + creates and manages pods, and can be autoscaled via HPA. +* `SandboxConfig`: declares sandbox types. Might be moved to the API server now + that fine-grained auth is supported through OpenFGA. Read directly by atelets. +* `CSIDriverConfig`: gives ATE direct access to CSI drivers without using + PV/PVCs. Read directly by atelets. + +In addition, ATE uses Secrets to project trust bundles into Actors, +`StorageClass` to set up access to storage, and `PodCertificateRequest` to +provision SPIFFE mTLS identities to workers. Finally, the atelet is deployed as +a daemonset on all nodes. + +While failure conditions may be displayed on CRD status fields, these are hard +to monitor and do not cover all the cases listed above. As a result, the primary +signal for failures with K8s resources and their controllers will be error logs, +unless otherwise listed below. + +### \[ICT, OPT\] B.1: Workers cannot be created + +K8s may not be able to schedule the requested pods for workers, which could be a +tradeoff issue (e.g. the cluster is not large enough). But there are many other +reasons the number of workers may not be at intent: for example, the +`WorkerPool` reconciler could fail to create the associated `Deployment`, or the +pods could be created but not noticed by the rest of the system due to a faulty +network policy. These are always incidents. + +We already track the two necessary metrics for this scenario: + +* `ate.workerpool.desired_workers` is emitted as soon as the data is read from + the WorkerPool CRD, so will be accurate as long as the controller is running + at all. +* `ate.workerpool.workers` is emitted once workers are actually registered with + ateapi. + +Since the intended worker pool size can change over time, the following alert +verifies that the current size is \>= the smallest requested size over the last +five minutes, and fires an alert after five minutes if not. While this will not +catch failures to *release* workers, this seems like a far less likely problem. + +**Example:** In the [sample rules](examples/gmp-alerting-rules.yaml), see +`ate_workerpool_alerts`. + +### \[ICT\] B.2: Existing workers become unreachable + +If ateapi stops being able to reach the atelet *after* creation \- for example, +because the atelet has crashed, or because a new NetworkPolicy breaks +communication \- there is currently no metric that will move (as of Oct 2026), +until a client attempts to contact it directly or indirectly via +`ate-api-server` (at which point the client will receive `UNAVAILABLE`, which is +caught by A.2). + +Worse, because ateapi assigns an actor to a worker before attempting to contact +the atelet, and because it treats `Unavailable` as a transient error, any actor +assigned to an unreachable worker will fail permanently. The worker doesn't +become a traditional black hole \- it will only "accept" as many actors as it +originally had capacity for \- but once the actor is removed, the zombie worker will +trap *future* actors. + +**TODO:** consider adding a heartbeat between ateapi and atelets; after a period +of time, declare the worker lost, decrease ate.workerpool.workers (so the alert +from B.1 fires) and stop assigning new actors to it. We could also wait for +external RPCs instead of heartbeats combined with a timeout or failure count. + +### \[ICT\] B.3: The atelet (or other components) experiences local failures + +The atelet can fail in any number of ways which are not necessarily exposed to +the API server RPCs. For example: + +* It could fail to receive the capacity or status of the worker or register this + information back to the API server. This could silently prevent the API server + from attempting to use the worker, without causing any RPC to fail. +* The atelet could fail to clean up after an actor eviction or to prune local + snapshots. + +In addition, components should emit errors if there is an *impending* failure. +For example, a certificate used for mTLS that is expiring in fewer than M days +very likely indicates that the certificate distribution system has broken down, +and should be exposed as an error (not warning) message. + +**Signal and alerting**: error logs. If ignored, these will eventually reduce +capacity, leading to violations of A.1. + +### \[ICT, OPT\] B.4: The atelet experiences performance degradations + +The atelet is responsible for storing and retrieving OCI images, actor +snapshots, and other assets. If these operations fail completely, the failures +will be detected via other surfaces (e.g. A.1 and B.3 at a minimum), but if +they're merely slower than expected, we would need dedicated signals to detect +the problem. + +Unfortunately, pure durations (e.g. `{"ate.actor.checkpoint.duration", +"ate.snapshot.phase"="persist"}`) are not an adequate metric, since assets can +vary in size by orders of magnitude. So we need a measure of throughput that's +resistant to very small assets. One possible approach is: + +* Batch together all assets so we're not measuring very small files +* Only calculate throughput for batches larger than the expected + [bandwidth-delay + product](https://en.wikipedia.org/wiki/Bandwidth-delay_product). For example, + for a minimum delay of 25ms and an expected throughput of 200MiB/S, we would + only include batches larger than \~5MiB. + +**Signals and alerting:** Add throughput metrics and set up the associated SLOs +and SLO-based alerting. This is low priority at this time. + +## C: Actor execution + +For the most part, ATE is not responsible for ensuring that actors can execute +correctly; this is primarily the responsibility of the owners of the actors +(e.g. the authors of the code and whomever sets resource requests) and the +sandboxes themselves. However, there are some possible failures from ATE itself, +plus ATE operators must be able to monitor the operation of actors. + +### \[OPT\] C.1: Actors do not have enough resources (e.g. CPU or RAM) + +**Signal:** metrics showing CPU usage, RAM usage, and counters showing the +number of OOMs. Many of these are available in the +[metrics](metrics/registry/metrics.yaml) and [events](metrics/registry/events.yaml) registries, +such as `ate.actor.stats` and `ate.actor.usage_sampled`. Also see the +[observability](observability.md) docs for information on log attributes such as +`ate.sandbox.oom_kills`. + +**Alerting:** n/a. Operators should monitor these signals for overall system +health, and *may* set SLOs on the percentage of successful executions if they +also control the workloads themselves, but resource exhaustion does not indicate +a problem with ATE. + +### \[ICT\] C.2: Ateworker cannot access the probers + +**Signal and alerting:** Error logs. If ignored, see C.4. + +### \[ICT\] C.3: Ateworker cannot receive and store stdout/stderr + +**Signal and alerting:** Error logs. No backup if ignored. + +### \[ICT\] C.4: Actors are failing for other reasons. + +E.g. atelets could fail to mount/unmount drives, or any other kind of local +failure. + +**Signal:** See C.1, and specifically the `ate.actor.crashes` metric and +`ate.actor.crashed` OTLP log event. + +**Alerting:** n/a + +## D: Actor network requests + +### \[ICT\] D.1: Inbound network requests cannot be routed + +End-to-end ingress routing success can be measured by Envoy using +`envoy_http_downstream_rq_total` and `envoy_http_downstream_rq_xx`. +Unfortunately, if the actor returns a 5xx status code, Envoy can't distinguish +between an error from ATE and from the Actor, so these are coarse signals that +shouldn't be used for SLOs. By contrast, `atenet.router.route.duration` contains +all the outcomes (see [observability](observability.md)) which is a more +reliable SLI, but may be missing some failure cases. + +### \[ICT\] D.2: Outbound network requests cannot be routed + +TBD + +# Appendix: monitoring ATE and defending SLOs + +"Defending" an SLO on ATE means: + +1. Monitoring its SLIs and [alerting on potential +violations](https://sre.google/workbook/alerting-on-slos/); +2. Taking actions to resolve the violations. + +In a self-managed deployment such as ATE, it is obviously not possible for +Google (or any other cloud provider) to take actions to resolve violations on +user-owned systems. For example, Google cannot: + +* Increase the number of workers (or underlying nodes) to increase capacity +* Roll back a problematic change +* Ban or rate-limit problematic clients + +There may be a certain class of problems that cloud providers *could* resolve, +but it's hard to say what those might be with any confidence, and it's highly +likely that these problems would not be specific to ATE anyway. + +In addition, even centralized monitoring is not possible. Google generally only +has access to metrics from Google-owned components (aka "system metrics"), but +since an ATE deployment is not a Google-owned component, its metrics are not +accessible to Google, [except under exceptional +conditions](https://cloud.google.com/security/products/access-transparency). +There could be a theoretical path by which we ask customers to provide +permission for Google to view these metrics, but they could always say no. These +restrictions presumably apply to other cloud providers as well. + +For this reason, the best we can do is to: + +* Give users useful SLIs so they can defend their own SLOs; +* Test ATE in controlled environments to maximize the likelihood that users will + be able to successfully defend their own SLOs, e.g. through performance + optimization. From 45ab64df295b6b635d113c1925bff27ae8c3535d Mon Sep 17 00:00:00 2001 From: Luiz Oliveira Date: Wed, 28 Oct 2026 18:01:00 +0000 Subject: [PATCH 36/53] The template reconciler skips templates marked for deletion (#3342) #3333 Signed-off-by: Luiz Oliveira --- cmd/ateapi/internal/controlapi/template_reconciler.go | 7 +++++-- .../internal/controlapi/template_reconciler_test.go | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 810ca27682..2f0c979145 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -197,8 +197,11 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource } return 0, err } - // TODO: Skip a template with delete_time set. DeleteActorTemplate may - // already have deleted the golden actor and tag, and this pass rebuilds them. + // DeleteActorTemplate may already have deleted the golden actor and tag, + // and this pass would rebuild them. + if tmpl.GetMetadata().GetDeleteTime() != nil { + return 0, nil + } if tmpl.GetSnapshotConfig().GetGoldenSnapshotConfig().GetMode() == ateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED { return 0, nil diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index dfac9b56e5..c2eea80cc0 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -352,6 +352,10 @@ func withoutWakeupProbe(tmpl *ateapi.ActorTemplate) { } } +func withDeleteTime(tmpl *ateapi.ActorTemplate) { + tmpl.Metadata.DeleteTime = timestamppb.Now() +} + func withoutGoldenSnapshot(tmpl *ateapi.ActorTemplate) { tmpl.SnapshotConfig.GoldenSnapshotConfig.Mode = ateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED } @@ -578,6 +582,12 @@ func TestReconcileOne(t *testing.T) { name: "deleted template is a noop", control: &fakeGoldenControl{}, }, + { + name: "template marked for deletion is a noop", + template: testTemplate(withDeleteTime), + control: &fakeGoldenControl{}, + wantUnchanged: true, + }, { name: "terminal golden snapshot is a noop", template: testTemplate(withGoldenTag()), From e8cdf13daf921520695db5ebd1999b6c2f739662 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Wed, 28 Oct 2026 18:41:14 +0000 Subject: [PATCH 37/53] Suspending actors no longer OOM-kills tight worker pods (#3353) A suspend writes about as much page cache as the actor has in memory, and it's charged to the worker pod. In a pod sized close to its actors that runs the pod out of memory, and the kernel OOM kills the worker along with every actor on it. Part of #3092. - While a microVM snapshot or gVisor checkpoint is written, its cgroup gets `memory.high`, so the page cache can only grow into what the actor booked and isn't using, plus what the pod has spare to lend. If the pod can lend all of it there's no limit, so pods with room still suspend at full speed. Tight pods slow down to disk speed instead. - A worker writes at most 2 snapshots at once, and a newly placed actor waits for lent memory to be written back before it boots - The rootfs upper and durable-dir tars write out and drop their page cache every 8MiB, so they can't pile up dirty pages either. This waits on the disk, so it gives back some of the speed #3026 got by dropping the fsync. Happy to tune it. - microVM actor overhead goes from 128MiB back to 4MiB + 8MiB/GiB, the 128MiB in #3104 was headroom for these writes. Both workers keep 32MiB plus 16MiB per write slot (64MiB) for themselves. Tested on GKE with a fresh install: - A 1GiB microVM actor filled to 893MiB in a 1132Mi pod survived 4/4 suspend/resume cycles - four 256Mi gVisor actors filled to 225MiB in a 1088Mi pod survived 6/6 concurrent suspend waves. - Without the cap that gVisor case OOM-killed the worker. Neither worker restarted. AI use: AI-assisted, I reviewed and tested it. - [x] Tests pass - [x] Docs updated --------- Signed-off-by: Benjamin Elder --- cmd/ateworker-gvisor/ckptcgroup.go | 27 +++- cmd/ateworker-gvisor/ckptcgroup_test.go | 43 ++++-- cmd/ateworker-gvisor/hosted.go | 13 +- cmd/ateworker-gvisor/hosted_test.go | 21 ++- cmd/ateworker-gvisor/main.go | 72 +++++++--- cmd/ateworker-gvisor/memory.go | 72 ++++++++++ cmd/ateworker-microvm/checkpoint.go | 72 +++++++++- cmd/ateworker-microvm/hosted.go | 15 ++- cmd/ateworker-microvm/hosted_test.go | 6 +- cmd/ateworker-microvm/main.go | 33 +++-- cmd/ateworker-microvm/net_linux_test.go | 2 +- cmd/ateworker-microvm/restore.go | 7 +- cmd/ateworker-microvm/run.go | 19 +-- cmd/ateworker-microvm/snapshotborrow.go | 37 ++++++ cmd/ateworker-microvm/snapshotborrow_test.go | 32 +++++ cmd/ateworker-microvm/snapshotcache.go | 7 +- internal/ateworker/cgroup/actor.go | 83 ++++++++++++ internal/ateworker/cgroup/actor_test.go | 95 +++++++++++++ internal/ateworker/memborrow/memborrow.go | 125 ++++++++++++++++++ .../ateworker/memborrow/memborrow_test.go | 109 +++++++++++++++ internal/snapshotcache/snapshotcache.go | 39 ++++++ internal/tarutil/tarutil.go | 30 ++++- internal/tarutil/tarutil_test.go | 28 ++++ 23 files changed, 904 insertions(+), 83 deletions(-) create mode 100644 cmd/ateworker-gvisor/memory.go create mode 100644 cmd/ateworker-microvm/snapshotborrow.go create mode 100644 cmd/ateworker-microvm/snapshotborrow_test.go create mode 100644 internal/ateworker/memborrow/memborrow.go create mode 100644 internal/ateworker/memborrow/memborrow_test.go diff --git a/cmd/ateworker-gvisor/ckptcgroup.go b/cmd/ateworker-gvisor/ckptcgroup.go index f2a5a6055f..51c3a0c99f 100644 --- a/cmd/ateworker-gvisor/ckptcgroup.go +++ b/cmd/ateworker-gvisor/ckptcgroup.go @@ -62,8 +62,9 @@ func checkpointCgroupDir(root, actorUID string) string { // isolateCheckpointCache moves the actor's sentry from its sandbox leaf into a // sibling -_ckpt cgroup, preserving the sandbox leaf's cpu.max so checkpoint -// serialization remains capped to the actor's CPU limit. -func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actorUID string) (*checkpointCgroup, error) { +// serialization remains capped to the actor's CPU limit. cacheHigh, when set, +// is the sibling's memory.high. +func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actorUID string, cacheHigh int64) (*checkpointCgroup, error) { pid, err := readSandboxPID(actorDirs) if err != nil { return nil, err @@ -82,6 +83,14 @@ func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actor slog.Debug("Failed to copy cpu.max to checkpoint cgroup", slog.String("cgroup", dir), slog.Any("err", err)) } } + // written even when unset, so a leftover cgroup does not keep an old limit + high := "max" + if cacheHigh > 0 { + high = strconv.FormatInt(cacheHigh, 10) + } + if err := writeExisting(filepath.Join(dir, "memory.high"), high); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("while limiting cgroup %q: %w", dir, err) + } c := &checkpointCgroup{leaf: leaf, dir: dir, pid: pid} if err := movePID(dir, pid); err != nil { return nil, errors.Join(err, c.remove()) @@ -89,6 +98,17 @@ func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actor return c, nil } +// writeExisting writes a cgroup interface file without creating it, so a node +// that did not delegate the controller has nothing written. +func writeExisting(path, value string) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0) + if err != nil { + return err + } + _, err = f.WriteString(value) + return errors.Join(err, f.Close()) +} + func readSandboxPID(actorDirs *ateworkerpb.ActorDirs) (int, error) { pidFile := pidFilePath(actorDirs, ocispec.SandboxID) b, err := os.ReadFile(pidFile) @@ -156,11 +176,12 @@ func (c *checkpointCgroup) remove() error { func checkpointIsolated( ctx context.Context, cgroupRoot, actorUID string, + cacheHigh int64, actorDirs *ateworkerpb.ActorDirs, checkpoint func(context.Context) error, terminate func() error, ) error { - ckptCgroup, err := isolateCheckpointCache(cgroupRoot, actorDirs, actorUID) + ckptCgroup, err := isolateCheckpointCache(cgroupRoot, actorDirs, actorUID, cacheHigh) if err != nil { slog.WarnContext(ctx, "Checkpointing without page cache isolation", slog.String("actorUID", actorUID), slog.Any("err", err)) diff --git a/cmd/ateworker-gvisor/ckptcgroup_test.go b/cmd/ateworker-gvisor/ckptcgroup_test.go index 910c98987e..ae2127f91a 100644 --- a/cmd/ateworker-gvisor/ckptcgroup_test.go +++ b/cmd/ateworker-gvisor/ckptcgroup_test.go @@ -106,7 +106,7 @@ func mustRead(t *testing.T, path string) string { func TestIsolateCheckpointCacheMovesOnlyTheSentry(t *testing.T) { f := newCkptFixture(t, "101") - c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID) + c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 0) if err != nil { t.Fatalf("isolateCheckpointCache: %v", err) } @@ -135,7 +135,7 @@ func TestIsolateCheckpointCacheMovesOnlyTheSentry(t *testing.T) { func TestIsolateCheckpointCacheWithoutPIDFile(t *testing.T) { f := newCkptFixture(t, "") - if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID); err == nil { + if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 0); err == nil { t.Fatal("isolateCheckpointCache succeeded with no sandbox pid file") } if _, err := os.Stat(checkpointCgroupDir(f.root, testActorUID)); !os.IsNotExist(err) { @@ -147,7 +147,7 @@ func TestIsolateCheckpointCacheRejectsPIDOutsideSandboxLeaf(t *testing.T) { f := newCkptFixture(t, "101") mustWrite(t, filepath.Join(f.leaf, "cgroup.procs"), "100\n") - if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID); err == nil { + if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 0); err == nil { t.Fatal("isolateCheckpointCache succeeded with the pid outside the sandbox leaf") } if _, err := os.Stat(checkpointCgroupDir(f.root, testActorUID)); !os.IsNotExist(err) { @@ -160,7 +160,7 @@ func TestIsolateCheckpointCacheRejectsPIDOutsideSandboxLeaf(t *testing.T) { func TestCheckpointCgroupRestoreAfterLeafGone(t *testing.T) { f := newCkptFixture(t, "101") - c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID) + c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 0) if err != nil { t.Fatalf("isolateCheckpointCache: %v", err) } @@ -179,7 +179,7 @@ func TestCheckpointCgroupRestoreAfterLeafGone(t *testing.T) { func TestCheckpointCgroupRestoreSkipsExitedSentry(t *testing.T) { f := newCkptFixture(t, "101") - c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID) + c, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 0) if err != nil { t.Fatalf("isolateCheckpointCache: %v", err) } @@ -221,7 +221,7 @@ func TestCheckpointIsolated(t *testing.T) { err := checkpointIsolated( context.Background(), - f.root, testActorUID, f.actorDirs, + f.root, testActorUID, 0, f.actorDirs, func(context.Context) error { if got := mustRead(t, filepath.Join(ckptDir, "cgroup.procs")); got != "101" { t.Errorf("checkpoint cgroup.procs during checkpoint = %q, want 101", got) @@ -253,7 +253,7 @@ func TestCheckpointIsolated(t *testing.T) { err := checkpointIsolated( context.Background(), - f.root, testActorUID, f.actorDirs, + f.root, testActorUID, 0, f.actorDirs, func(context.Context) error { if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { t.Errorf("checkpoint cgroup should not be created without a sandbox pid: %v", err) @@ -280,7 +280,7 @@ func TestCheckpointIsolated(t *testing.T) { err := checkpointIsolated( context.Background(), - f.root, testActorUID, f.actorDirs, + f.root, testActorUID, 0, f.actorDirs, func(context.Context) error { return errors.New("checkpoint boom") }, @@ -310,7 +310,7 @@ func TestCheckpointIsolated(t *testing.T) { err := checkpointIsolated( context.Background(), - f.root, testActorUID, f.actorDirs, + f.root, testActorUID, 0, f.actorDirs, func(context.Context) error { movePID = func(string, int) error { return unix.EACCES } return errors.New("checkpoint boom") @@ -339,7 +339,7 @@ func TestCheckpointIsolated(t *testing.T) { err := checkpointIsolated( context.Background(), - f.root, testActorUID, f.actorDirs, + f.root, testActorUID, 0, f.actorDirs, func(context.Context) error { if err := os.RemoveAll(f.leaf); err != nil { t.Fatal(err) @@ -369,7 +369,7 @@ func TestCheckpointIsolated(t *testing.T) { err := checkpointIsolated( context.Background(), - f.root, testActorUID, f.actorDirs, + f.root, testActorUID, 0, f.actorDirs, func(context.Context) error { return nil }, func() error { // Non-empty ckptDir emulates a sentry still alive in the cgroup. @@ -616,3 +616,24 @@ func TestEvictPreservedRestoreDir(t *testing.T) { } }) } + +func TestIsolateCheckpointCacheLimitsTheCache(t *testing.T) { + f := newCkptFixture(t, "101") + // cgroupfs provides memory.high itself + dir := checkpointCgroupDir(f.root, testActorUID) + mustMkdir(t, dir) + mustWrite(t, filepath.Join(dir, "memory.high"), "max\n") + if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 64<<20); err != nil { + t.Fatalf("isolateCheckpointCache: %v", err) + } + if got := mustRead(t, filepath.Join(dir, "memory.high")); got != "67108864" { + t.Errorf("checkpoint memory.high = %q, want 67108864", got) + } + // a leftover cgroup reused without a limit drops the old one + if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID, 0); err != nil { + t.Fatalf("isolateCheckpointCache: %v", err) + } + if got := mustRead(t, filepath.Join(dir, "memory.high")); got != "max" { + t.Errorf("checkpoint memory.high = %q, want max", got) + } +} diff --git a/cmd/ateworker-gvisor/hosted.go b/cmd/ateworker-gvisor/hosted.go index 47e9b56e1d..239be919df 100644 --- a/cmd/ateworker-gvisor/hosted.go +++ b/cmd/ateworker-gvisor/hosted.go @@ -55,6 +55,8 @@ type hostedActor struct { // nothing that waits on it later can be fooled by a reused pid. Nil if it // could not be pinned. Set once, under the actor's lock. sandbox *sandboxProcess + // memoryBytes is the actor's memory limit, what it books on the worker. + memoryBytes int64 // memoryLimit is the sandbox leaf's memory.high, or 0 while watchMemory // should leave it alone. memoryLimit atomic.Uint64 @@ -68,7 +70,7 @@ type hostedActor struct { // admitActor reserves capacity before network setup. An actor that is already // hosted keeps its slot; its old network is returned for the caller to close. -func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution) (*hostedActor, *actornet.SandboxSession, error) { +func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution, memoryBytes int64) (*hostedActor, *actornet.SandboxSession, error) { s.actorsMu.Lock() defer s.actorsMu.Unlock() var stale *actornet.SandboxSession @@ -87,16 +89,17 @@ func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution) (* // The sandbox cgroup is new with every activation, so its counters start // at zero. ctx, cancel := context.WithCancel(context.Background()) - hosted := &hostedActor{attribution: attribution, usage: workerstats.NewActivation(time.Now(), false), ctx: ctx, cancel: cancel} + hosted := &hostedActor{attribution: attribution, usage: workerstats.NewActivation(time.Now(), false), ctx: ctx, cancel: cancel, memoryBytes: memoryBytes} s.actors[attribution.UID] = hosted return hosted, stale, nil } -// hostActor sets up the actor's network, replacing any stale one. -func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources.ActorAttribution, actorDirs *ateworkerpb.ActorDirs) (*hostedActor, error) { +// hostActor sets up the actor's network, replacing any stale one. memoryBytes +// is what the actor books. +func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources.ActorAttribution, actorDirs *ateworkerpb.ActorDirs, memoryBytes int64) (*hostedActor, error) { uid := attribution.UID - hosted, stale, err := s.admitActor(attribution) + hosted, stale, err := s.admitActor(attribution, memoryBytes) if err != nil { return nil, err } diff --git a/cmd/ateworker-gvisor/hosted_test.go b/cmd/ateworker-gvisor/hosted_test.go index e3aebb6594..646fd62132 100644 --- a/cmd/ateworker-gvisor/hosted_test.go +++ b/cmd/ateworker-gvisor/hosted_test.go @@ -43,7 +43,7 @@ func TestAdmitActorEnforcesTheCeilingConcurrently(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - _, _, _ = s.admitActor(resources.ActorAttribution{UID: fmt.Sprintf("actor-%d", i)}) + _, _, _ = s.admitActor(resources.ActorAttribution{UID: fmt.Sprintf("actor-%d", i)}, 0) }() } wg.Wait() @@ -57,11 +57,11 @@ func TestAdmitActorEnforcesTheCeilingConcurrently(t *testing.T) { // on a full worker and a retry cannot lose its place to another actor. func TestAdmitActorKeepsAHostedActorsSlot(t *testing.T) { s := &AteWorkerService{actors: map[string]*hostedActor{}, maxActors: 1} - first, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}) + first, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, 0) if err != nil { t.Fatal(err) } - again, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}) + again, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, 0) if err != nil { t.Fatalf("re-admitting a hosted actor on a full worker: %v", err) } @@ -69,7 +69,7 @@ func TestAdmitActorKeepsAHostedActorsSlot(t *testing.T) { t.Error("re-admission kept the old record; readers could not tell the incarnations apart") } // ResourceExhausted, so the control plane treats it as a capacity miss. - if _, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-b"}); apierror.Code(err) != codes.ResourceExhausted { + if _, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-b"}, 0); apierror.Code(err) != codes.ResourceExhausted { t.Errorf("admitting past the ceiling: got %v, want ResourceExhausted", err) } if got := len(s.hostedActors()); got != 1 { @@ -85,7 +85,7 @@ func TestDrainingActorsStillCountAgainstTheCeiling(t *testing.T) { maxActors: 1, draining: 1, } - if _, err := s.hostActor(context.Background(), resources.ActorAttribution{UID: "actor-a"}, nil); err == nil { + if _, err := s.hostActor(context.Background(), resources.ActorAttribution{UID: "actor-a"}, nil, 0); err == nil { t.Error("admitted an actor while a draining one still held the only place") } } @@ -165,3 +165,14 @@ func TestBeginRPCRegistersBeforeCheckingForADrain(t *testing.T) { t.Errorf("a refused RPC stayed registered: %v", got) } } + +func TestUnbookedMemory(t *testing.T) { + const limit = 4 << 30 + s := &AteWorkerService{actors: map[string]*hostedActor{ + "a": {memoryBytes: 1 << 30}, + "b": {memoryBytes: 256 << 20}, + }} + if got, want := s.unbookedMemory(limit), int64(limit-memoryReservedBytes-1<<30-256<<20); got != want { + t.Errorf("unbookedMemory() = %d, want %d", got, want) + } +} diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index 7f2ec9ff81..908afccf03 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -43,6 +43,7 @@ import ( "go.ate.dev/substrate/internal/ateworker" "go.ate.dev/substrate/internal/ateworker/actornet" "go.ate.dev/substrate/internal/ateworker/cgroup" + "go.ate.dev/substrate/internal/ateworker/memborrow" "go.ate.dev/substrate/internal/ateworker/phaselog" "go.ate.dev/substrate/internal/ateworker/pid1" "go.ate.dev/substrate/internal/ateworker/register" @@ -57,6 +58,7 @@ import ( "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/internal/serverboot" "go.ate.dev/substrate/internal/sizing" + "go.ate.dev/substrate/internal/snapshotcache" "go.ate.dev/substrate/internal/version" "go.ate.dev/substrate/internal/wakeupprobe" "go.ate.dev/substrate/internal/workerstats" @@ -92,13 +94,6 @@ var ( "Unix socket of atelet's OTLP relay to export telemetry through, keeping it off the pod network. Empty, or absent at startup, exports directly to OTEL_EXPORTER_OTLP_ENDPOINT instead.") ) -// memoryReservedBytes is how much of the worker pod's memory limit we keep from -// actors: the worker process itself, and the page cache of checkpoints in -// flight, which leaves the actor's own cgroup with its sentry (see -// isolateCheckpointCache). Otherwise an actor's sentry and gofers are charged -// to its own cgroup, so gVisor reports no actor overhead. -const memoryReservedBytes = 64 << 20 - // minUsageSampleInterval is the floor of --usage-sample-interval, the same as // atelet's poll interval floor. const minUsageSampleInterval = 50 * time.Second @@ -389,22 +384,30 @@ type AteWorkerService struct { // flags holds the watched JSON flag configuration. flags *flagStore + + // checkpointWrites holds a slot per checkpoint writing its page cache, + // which memoryReservedBytes budgets for. + checkpointWrites chan struct{} + borrow *memborrow.Ledger } var _ ateworkerpb.WorkerServer = (*AteWorkerService)(nil) // NewService creates a new AteWorkerService. func NewService(tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, maxActors int, flags *flagStore) *AteWorkerService { - return &AteWorkerService{ - locks: actorlock.New(), - inFlight: actorlock.NewInFlight(), - actors: map[string]*hostedActor{}, - maxActors: maxActors, - tunnel: tunnel, - actorLogger: actorLogger, - cgroupRoot: defaultCgroupRoot, - flags: flags, - } + s := &AteWorkerService{ + locks: actorlock.New(), + inFlight: actorlock.NewInFlight(), + actors: map[string]*hostedActor{}, + maxActors: maxActors, + tunnel: tunnel, + actorLogger: actorLogger, + cgroupRoot: defaultCgroupRoot, + flags: flags, + checkpointWrites: make(chan struct{}, checkpointWriteSlots), + } + s.borrow = &memborrow.Ledger{Unbooked: s.unbookedMemory} + return s } // beginRPC registers before checking the drain flag so shutdown cannot miss it. @@ -662,7 +665,7 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run return nil, err } // Publish attribution before boot so stats can include startup usage. - hosted, err := s.hostActor(ctx, attribution, req.GetActorDirs()) + hosted, err := s.hostActor(ctx, attribution, req.GetActorDirs(), sizing.FromLimits(req.GetResources().GetLimits()).MemoryBytes) if err != nil { return nil, err } @@ -700,6 +703,9 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run } } }() + if err := s.borrow.Wait(ctx); err != nil { + return nil, err + } // Egress is activated before the sandbox starts, ingress is activated after // the wakeup probe (see tunnel.Tunnel.ActivateEgress). if err := s.tunnel.ActivateEgress(attribution, egress); err != nil { @@ -927,15 +933,28 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke default: return nil, fmt.Errorf("unsupported snapshot fidelity: %v", fidelity) } + if s.checkpointWrites != nil { + select { + case s.checkpointWrites <- struct{}{}: + case <-ctx.Done(): + return nil, fmt.Errorf("while waiting to checkpoint: %w", ctx.Err()) + } + } // the sentry leaves its sandbox leaf for the checkpoint cgroup var memoryLimit uint64 if hosted != nil { memoryLimit = pauseMemoryWatch(hosted) } - if err := checkpointIsolated(ctx, s.cgroupRoot, req.GetActorUid(), req.GetActorDirs(), checkpointFn, terminate); err != nil { + cacheHigh, borrowed := s.checkpointCacheLimit(hosted, req.GetActorUid()) + err = checkpointIsolated(ctx, s.cgroupRoot, req.GetActorUid(), cacheHigh, req.GetActorDirs(), checkpointFn, terminate) + if s.checkpointWrites != nil { + <-s.checkpointWrites + } + if err != nil { if hosted != nil && memoryLimit != 0 { resumeMemoryWatch(hosted, memoryLimit) } + s.borrow.Return(borrowed) return nil, err } @@ -944,8 +963,18 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke // rather than a hardcoded list. snapshotFiles, err := listSnapshotFiles(checkpointPath) if err != nil { + s.borrow.Return(borrowed) return nil, fmt.Errorf("while listing checkpoint files: %w", err) } + // the borrowed page cache is returned once it is clean, and reclaimable + written := make([]string, len(snapshotFiles)) + for i, f := range snapshotFiles { + written[i] = filepath.Join(checkpointPath, f) + } + if err := snapshotcache.StartWriteback(written, func() { s.borrow.Return(borrowed) }); err != nil { + s.borrow.Return(borrowed) + slog.WarnContext(ctx, "Failed to start checkpoint writeback", slog.String("actorUID", req.GetActorUid()), slog.Any("err", err)) + } durableFiles := durableSnapshotFiles(snapshotFiles) s.actorLogger.EmitLifecycleLog(ctx, "Actor checkpointed", attribution) @@ -1112,7 +1141,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb if err != nil { return nil, err } - hosted, err := s.hostActor(ctx, attribution, req.GetActorDirs()) + hosted, err := s.hostActor(ctx, attribution, req.GetActorDirs(), sizing.FromLimits(req.GetResources().GetLimits()).MemoryBytes) timing.netSetup = lap(&tLast) if err != nil { return nil, err @@ -1148,6 +1177,9 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb } } }() + if err := s.borrow.Wait(ctx); err != nil { + return nil, err + } err = s.tunnel.ActivateEgress(attribution, egress) timing.activate = lap(&tLast) if err != nil { diff --git a/cmd/ateworker-gvisor/memory.go b/cmd/ateworker-gvisor/memory.go new file mode 100644 index 0000000000..4f1e93deaf --- /dev/null +++ b/cmd/ateworker-gvisor/memory.go @@ -0,0 +1,72 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "go.ate.dev/substrate/internal/ateworker/cgroup" + "go.ate.dev/substrate/internal/ocispec" +) + +// checkpointWriteHeadroom is how far a checkpoint's page cache may grow when +// the actor's booking leaves less than that, and checkpointWriteSlots how many +// checkpoints may write at once. memoryReservedBytes covers both. +const ( + checkpointWriteHeadroom = 16 << 20 + checkpointWriteSlots = 2 +) + +// memoryReservedBytes is what the worker keeps from actors: its own process, +// and the page cache of the checkpoints it lets write at once, which leaves +// the actor's cgroup with its sentry (see isolateCheckpointCache). +const memoryReservedBytes = 32<<20 + checkpointWriteSlots*checkpointWriteHeadroom + +// unbookedMemory is the worker's reported memory capacity less what its +// actors booked. +func (s *AteWorkerService) unbookedMemory(podLimit int64) int64 { + unbooked := podLimit - memoryReservedBytes + s.actorsMu.RLock() + defer s.actorsMu.RUnlock() + for _, h := range s.actors { + unbooked -= h.memoryBytes + } + return unbooked +} + +// checkpointCacheLimit returns the memory.high for the actor's checkpoint +// cgroup, 0 for none, and what it borrowed for it. +// +// The checkpoint writes about as much as the sandbox holds through the page +// cache, charged to the checkpoint cgroup. At the worker pod's memory limit +// that can OOM-kill the whole pod, so the cache may only grow into what the +// actor booked and the sandbox left untouched, plus what it can borrow. When +// the pod can lend all of it there is no limit: memory.high also shrinks the +// cgroup's dirty page limit, which throttles the write to the disk. +func (s *AteWorkerService) checkpointCacheLimit(hosted *hostedActor, actorUID string) (high, borrowed int64) { + used, err := cgroup.ActorLeafMemory(ocispec.GVisorCgroupLeaf(actorUID, ocispec.SandboxID)) + if err != nil { + return checkpointWriteHeadroom, 0 + } + borrowed = s.borrow.Take(used) + if borrowed >= used { + return 0, borrowed + } + var booked int64 + if hosted != nil { + booked = hosted.memoryBytes + } + return max(checkpointWriteHeadroom, booked-used) + borrowed, borrowed +} diff --git a/cmd/ateworker-microvm/checkpoint.go b/cmd/ateworker-microvm/checkpoint.go index ff1e178340..de567b50d9 100644 --- a/cmd/ateworker-microvm/checkpoint.go +++ b/cmd/ateworker-microvm/checkpoint.go @@ -31,10 +31,12 @@ import ( "go.ate.dev/substrate/cmd/ateworker-microvm/internal/ch" "go.ate.dev/substrate/cmd/ateworker-microvm/internal/kata" "go.ate.dev/substrate/internal/ateworker" + "go.ate.dev/substrate/internal/ateworker/cgroup" "go.ate.dev/substrate/internal/ateworker/phaselog" "go.ate.dev/substrate/internal/ateworkervalidation" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/proto/ateworkerpb" + "go.ate.dev/substrate/internal/snapshotcache" "go.ate.dev/substrate/internal/workerstats" "golang.org/x/sync/errgroup" ) @@ -288,8 +290,8 @@ func (s *AteWorkerService) snapshotVMState(ctx context.Context, client *ch.Clien slog.InfoContext(ctx, "Snapshotting guest", slog.String("id", actorUID), slog.String("dir", checkpointDir)) tSnapshot := time.Now() - if err := client.Snapshot(ctx, checkpointDir); err != nil { - return 0, fmt.Errorf("while snapshotting guest: %w", err) + if err := s.writeSnapshot(ctx, client, ra, actorUID, checkpointDir); err != nil { + return 0, err } dSnapshot := time.Since(tSnapshot) @@ -319,6 +321,72 @@ func (s *AteWorkerService) snapshotVMState(ctx context.Context, client *ch.Clien return dSnapshot, nil } +// snapshotWriteHeadroom is how far the actor's cgroup may grow past the paused +// guest while CH writes the snapshot, when its booking leaves less than that, +// and snapshotWriteSlots how many of those writes may run at once. +// memoryReservedBytes covers both. +const ( + snapshotWriteHeadroom = 16 << 20 + snapshotWriteSlots = 2 +) + +// writeSnapshot has CH write the paused guest into checkpointDir. +// +// CH writes through the page cache from the actor's cgroup while the guest +// still holds its memory. At the worker pod's memory limit that can OOM-kill +// the whole pod, and every actor on it, so the leaf may only grow into what the +// actor booked and the guest left untouched, plus what it can borrow, and the +// kernel throttles CH to the disk past that. +func (s *AteWorkerService) writeSnapshot(ctx context.Context, client *ch.Client, ra *runningActor, actorUID, checkpointDir string) error { + if s.snapshotWrites != nil { + select { + case s.snapshotWrites <- struct{}{}: + defer func() { <-s.snapshotWrites }() + case <-ctx.Done(): + return fmt.Errorf("while waiting to snapshot: %w", ctx.Err()) + } + } + var borrowed int64 + // the write caches about as much as the guest has populated, which is + // at most the leaf's usage. When the pod can lend all of that, the leaf + // stays unlimited: memory.high also shrinks the cgroup's dirty page + // limit, which throttles the write to the disk long before the cap + current, err := cgroup.ActorLeafMemory(actorUID) + if err == nil { + borrowed = s.borrow.Take(current) + } + if err != nil || borrowed < current { + var ceiling int64 + if ra != nil { + ceiling = ra.memoryBytes + } + release, err := cgroup.LimitActorLeafGrowth(actorUID, snapshotWriteHeadroom, ceiling, borrowed) + if err != nil { + slog.WarnContext(ctx, "Snapshotting without a page cache limit", slog.String("id", actorUID), slog.Any("err", err)) + } else { + defer release() + } + } + if err := client.Snapshot(ctx, checkpointDir); err != nil { + s.borrow.Return(borrowed) + return fmt.Errorf("while snapshotting guest: %w", err) + } + // clean pages can be reclaimed, and dropped once a restore has read them. + // the tars sync themselves + var written []string + for _, name := range []string{"config.json", "state.json", memoryRangesFile} { + p := filepath.Join(checkpointDir, name) + if _, err := os.Stat(p); err == nil { + written = append(written, p) + } + } + if err := snapshotcache.StartWriteback(written, func() { s.borrow.Return(borrowed) }); err != nil { + s.borrow.Return(borrowed) + slog.WarnContext(ctx, "Failed to start snapshot writeback", slog.String("id", actorUID), slog.Any("err", err)) + } + return nil +} + // mergeOnDemandDelta merges the OnDemand delta in checkpointDir with the base in // restoreSourceDir. A preserved base is copied, never modified in place. func mergeOnDemandDelta(ctx context.Context, restoreSourceDir, checkpointDir string, preserveRestoreSource bool) error { diff --git a/cmd/ateworker-microvm/hosted.go b/cmd/ateworker-microvm/hosted.go index 9b2fe6c69c..ec07b290c7 100644 --- a/cmd/ateworker-microvm/hosted.go +++ b/cmd/ateworker-microvm/hosted.go @@ -48,13 +48,16 @@ type hostedActor struct { // usage is this activation's epoch, CPU baseline, and latest sample. // Immutable after admission. usage *workerstats.Activation + // memoryBytes is the actor's memory limit, what it books on the worker. + // Immutable after admission. + memoryBytes int64 } // admitActor reserves capacity before network setup. An actor that is already // hosted keeps its slot; its old network is returned for the caller to close. // resumesGuest is true when the activation resumes a guest from a snapshot, -// whose CPU counters come back with it. -func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution, resumesGuest bool) (*hostedActor, *actornet.SandboxSession, error) { +// whose CPU counters come back with it. memoryBytes is what the actor books. +func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution, resumesGuest bool, memoryBytes int64) (*hostedActor, *actornet.SandboxSession, error) { s.actorsMu.Lock() defer s.actorsMu.Unlock() var stale *actornet.SandboxSession @@ -63,17 +66,17 @@ func (s *AteWorkerService) admitActor(attribution resources.ActorAttribution, re } else if len(s.actors)+s.draining >= s.maxActors { return nil, nil, apierror.ResourceExhausted("worker is full: %d actors", s.maxActors) } - hosted := &hostedActor{attribution: attribution, usage: workerstats.NewActivation(time.Now(), resumesGuest)} + hosted := &hostedActor{attribution: attribution, usage: workerstats.NewActivation(time.Now(), resumesGuest), memoryBytes: memoryBytes} s.actors[attribution.UID] = hosted return hosted, stale, nil } // hostActor sets up the actor's network, replacing any stale one. resumesGuest -// is as for admitActor. -func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources.ActorAttribution, resumesGuest bool) (*hostedActor, error) { +// and memoryBytes are as for admitActor. +func (s *AteWorkerService) hostActor(ctx context.Context, attribution resources.ActorAttribution, resumesGuest bool, memoryBytes int64) (*hostedActor, error) { uid := attribution.UID - hosted, stale, err := s.admitActor(attribution, resumesGuest) + hosted, stale, err := s.admitActor(attribution, resumesGuest, memoryBytes) if err != nil { return nil, err } diff --git a/cmd/ateworker-microvm/hosted_test.go b/cmd/ateworker-microvm/hosted_test.go index 1ce50ec51b..0a6b0039e6 100644 --- a/cmd/ateworker-microvm/hosted_test.go +++ b/cmd/ateworker-microvm/hosted_test.go @@ -29,11 +29,11 @@ import ( // on a full worker and a retry cannot lose its place to another actor. func TestAdmitActorKeepsAHostedActorsSlot(t *testing.T) { s := &AteWorkerService{actors: map[string]*hostedActor{}, maxActors: 1} - first, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, false) + first, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, false, 0) if err != nil { t.Fatal(err) } - again, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, false) + again, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, false, 0) if err != nil { t.Fatalf("re-admitting a hosted actor on a full worker: %v", err) } @@ -41,7 +41,7 @@ func TestAdmitActorKeepsAHostedActorsSlot(t *testing.T) { t.Error("re-admission kept the old record; readers could not tell the incarnations apart") } // ResourceExhausted, so the control plane treats it as a capacity miss. - if _, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-b"}, false); apierror.Code(err) != codes.ResourceExhausted { + if _, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-b"}, false, 0); apierror.Code(err) != codes.ResourceExhausted { t.Errorf("admitting past the ceiling: got %v, want ResourceExhausted", err) } if got := len(s.hostedActors()); got != 1 { diff --git a/cmd/ateworker-microvm/main.go b/cmd/ateworker-microvm/main.go index b93b575031..475bff12f4 100644 --- a/cmd/ateworker-microvm/main.go +++ b/cmd/ateworker-microvm/main.go @@ -45,6 +45,7 @@ import ( "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/ateworker/actornet" "go.ate.dev/substrate/internal/ateworker/cgroup" + "go.ate.dev/substrate/internal/ateworker/memborrow" "go.ate.dev/substrate/internal/ateworker/pid1" "go.ate.dev/substrate/internal/ateworker/register" "go.ate.dev/substrate/internal/atunnel" @@ -373,6 +374,11 @@ type AteWorkerService struct { chBinary string guestDebug bool + // snapshotWrites holds a slot per CH snapshot write in flight, whose page + // cache memoryReservedBytes budgets for. + snapshotWrites chan struct{} + borrow *memborrow.Ledger + // actorLogger forwards the actor container's stdout/stderr to the worker pod's // stdout as ate.dev/*-labeled JSON and emits actor lifecycle events (parity // with ateworker-gvisor). @@ -397,18 +403,21 @@ var _ ateworkerpb.WorkerServer = (*AteWorkerService)(nil) // NewService creates a new AteWorkerService. func NewService(podUID, chBinary string, guestDebug bool, maxActors int, tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger) *AteWorkerService { - return &AteWorkerService{ - locks: actorlock.New(), - inFlight: actorlock.NewInFlight(), - actors: map[string]*hostedActor{}, - guestSlots: make(chan struct{}, statsFanOut), - maxActors: maxActors, - podUID: podUID, - chBinary: chBinary, - guestDebug: guestDebug, - tunnel: tunnel, - actorLogger: actorLogger, - } + s := &AteWorkerService{ + locks: actorlock.New(), + inFlight: actorlock.NewInFlight(), + actors: map[string]*hostedActor{}, + guestSlots: make(chan struct{}, statsFanOut), + maxActors: maxActors, + podUID: podUID, + chBinary: chBinary, + guestDebug: guestDebug, + snapshotWrites: make(chan struct{}, snapshotWriteSlots), + tunnel: tunnel, + actorLogger: actorLogger, + } + s.borrow = &memborrow.Ledger{Unbooked: s.unbookedMemory} + return s } // beginRPC registers before checking the drain flag so shutdown cannot miss it. diff --git a/cmd/ateworker-microvm/net_linux_test.go b/cmd/ateworker-microvm/net_linux_test.go index a6e156cc00..69e27140a4 100644 --- a/cmd/ateworker-microvm/net_linux_test.go +++ b/cmd/ateworker-microvm/net_linux_test.go @@ -53,7 +53,7 @@ func TestHostActorReplacesSameActor(t *testing.T) { } }) for range 2 { - if _, err := service.hostActor(ctx, resources.ActorAttribution{UID: actorUID}, false); err != nil { + if _, err := service.hostActor(ctx, resources.ActorAttribution{UID: actorUID}, false, 0); err != nil { t.Fatal(err) } } diff --git a/cmd/ateworker-microvm/restore.go b/cmd/ateworker-microvm/restore.go index fae3772575..f1677ba808 100644 --- a/cmd/ateworker-microvm/restore.go +++ b/cmd/ateworker-microvm/restore.go @@ -175,7 +175,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb // so only it counts from the first reading; the lower fidelities cold-boot // a fresh guest whose CPU counts from zero. resumesGuest := includesMemory(req.GetFidelity()) - if _, err := s.hostActor(ctx, attribution, resumesGuest); err != nil { + if _, err := s.hostActor(ctx, attribution, resumesGuest, p.size.MemoryBytes); err != nil { return nil, err } defer func() { @@ -186,6 +186,9 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb _ = s.unhostActor(cleanupCtx, attribution.UID) } }() + if err := s.borrow.Wait(ctx); err != nil { + return nil, err + } // Restore the durable-dir volumes before anything can observe them: for MEMORY // that means before the share's virtiofsd starts, for the lower fidelities @@ -521,7 +524,7 @@ func (s *AteWorkerService) restoreMemoryFidelity(ctx context.Context, p actorBoo ra := &runningActor{ chProc: chProc, vfsdProc: vfsdProc, - apiSocket: apiSocket, baseID: srcID, restoreSourceDir: restoreDir, + apiSocket: apiSocket, memoryBytes: p.size.MemoryBytes, baseID: srcID, restoreSourceDir: restoreDir, preserveRestoreSource: preserveRestoreDir, snapshotIsSelfContained: memMode == ch.MemRestoreEager, // Signaling an id the agent does not know fails the whole graceful diff --git a/cmd/ateworker-microvm/run.go b/cmd/ateworker-microvm/run.go index ffc403e709..351203e970 100644 --- a/cmd/ateworker-microvm/run.go +++ b/cmd/ateworker-microvm/run.go @@ -73,6 +73,8 @@ type runningActor struct { // exit from a crash: shutting the VM down makes cloud-hypervisor exit, and // virtiofsd exits once its only client is gone. stopping atomic.Bool + // memoryBytes is the actor's memory limit, what it books on the worker. + memoryBytes int64 // restoreSourceDir is the snapshot dir this actor was OnDemand-restored from // (CH demand-pages its guest RAM from it). Set when restored via OnDemand. @@ -140,16 +142,12 @@ const kataAgentPath = "/usr/bin/kata-agent" // pod cgroup, which the control plane books with every actor: cloud-hypervisor // and virtiofsd, and the host page tables for guest RAM, KVM's EPT and the // memfd's xarray. -// -// The fixed part also covers the page cache of the actor's snapshot writes, -// which are not capped yet, so tight pods don't OOM on suspend. -// TODO: drop that part once snapshot writes are capped -var actorOverhead = &ateletpb.RuntimeOverhead{MemoryBytes: 128 << 20, MemoryBytesPerGib: 8 << 20} +var actorOverhead = &ateletpb.RuntimeOverhead{MemoryBytes: 4 << 20, MemoryBytesPerGib: 8 << 20} // memoryReservedBytes is how much of the worker pod's memory limit we keep from // actors: the worker process itself, and the page cache of snapshot writes in // flight. -const memoryReservedBytes = 64 << 20 +const memoryReservedBytes = 32<<20 + snapshotWriteSlots*snapshotWriteHeadroom // minGuestMemMiB is the floor for guest RAM, and so for an actor's memory limit. A // smaller one is rejected at cold boot (see resolveGuestMemMiB), since too little @@ -278,7 +276,7 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run return nil, err } // Publish attribution before boot so stats can include startup usage. - if _, err := s.hostActor(ctx, attribution, false); err != nil { + if _, err := s.hostActor(ctx, attribution, false, p.size.MemoryBytes); err != nil { return nil, err } defer func() { @@ -289,6 +287,9 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run _ = s.unhostActor(cleanupCtx, attribution.UID) } }() + if err := s.borrow.Wait(ctx); err != nil { + return nil, err + } if err := s.coldBootActorRetrying(ctx, p); err != nil { return nil, err @@ -369,7 +370,7 @@ func (s *AteWorkerService) coldBootActorRetrying(ctx context.Context, p actorBoo slog.String("id", p.actorUID), slog.Int("attempt", attempt), slog.Any("err", err)) // The failed attempt deactivated egress, which retires the listener // bound to it, so the network is rebuilt. The actor keeps its slot. - if _, err := s.hostActor(ctx, p.attribution(), false); err != nil { + if _, err := s.hostActor(ctx, p.attribution(), false, p.size.MemoryBytes); err != nil { return err } } @@ -607,7 +608,7 @@ func (s *AteWorkerService) coldBootActor(ctx context.Context, p actorBootParams) slog.Duration("wakeup_probe", time.Since(tContainers)), slog.Duration("since_boot", time.Since(tBooted))) - ra := &runningActor{chProc: chProc, vfsdProc: vfsdProc, apiSocket: apiSocket, baseID: actorUID, guestAgent: ac, workloadIDs: workloadIDs(ctrs)} + ra := &runningActor{chProc: chProc, vfsdProc: vfsdProc, apiSocket: apiSocket, memoryBytes: p.size.MemoryBytes, baseID: actorUID, guestAgent: ac, workloadIDs: workloadIDs(ctrs)} if err := s.tunnel.ActivateIngress(p.attribution(), s.sandboxDialer(p.actorUID)); err != nil { return err } diff --git a/cmd/ateworker-microvm/snapshotborrow.go b/cmd/ateworker-microvm/snapshotborrow.go new file mode 100644 index 0000000000..da5868028a --- /dev/null +++ b/cmd/ateworker-microvm/snapshotborrow.go @@ -0,0 +1,37 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// unbookedMemory is the worker's reported memory capacity less what its +// actors booked, overhead included. +func (s *AteWorkerService) unbookedMemory(podLimit int64) int64 { + unbooked := podLimit - memoryReservedBytes + s.actorsMu.RLock() + defer s.actorsMu.RUnlock() + for _, h := range s.actors { + unbooked -= actorBooking(h.memoryBytes) + } + return unbooked +} + +// actorBooking is what the control plane books for an actor with this memory +// limit: the limit plus actorOverhead. +func actorBooking(memoryBytes int64) int64 { + const gib = 1 << 30 + perGiB := actorOverhead.GetMemoryBytesPerGib() + return memoryBytes + actorOverhead.GetMemoryBytes() + memoryBytes/gib*perGiB + memoryBytes%gib*perGiB/gib +} diff --git a/cmd/ateworker-microvm/snapshotborrow_test.go b/cmd/ateworker-microvm/snapshotborrow_test.go new file mode 100644 index 0000000000..6f0312d7c2 --- /dev/null +++ b/cmd/ateworker-microvm/snapshotborrow_test.go @@ -0,0 +1,32 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "testing" + +func TestUnbookedMemory(t *testing.T) { + const limit = 4 << 30 + s := &AteWorkerService{actors: map[string]*hostedActor{ + "a": {memoryBytes: 1 << 30}, + "b": {memoryBytes: 512 << 20}, + }} + // 4MiB each, plus 8MiB per GiB + want := int64(limit - memoryReservedBytes - (1<<30 + 4<<20 + 8<<20) - (512<<20 + 4<<20 + 4<<20)) + if got := s.unbookedMemory(limit); got != want { + t.Errorf("unbookedMemory() = %d, want %d", got, want) + } +} diff --git a/cmd/ateworker-microvm/snapshotcache.go b/cmd/ateworker-microvm/snapshotcache.go index 91ab2618ae..971f540a33 100644 --- a/cmd/ateworker-microvm/snapshotcache.go +++ b/cmd/ateworker-microvm/snapshotcache.go @@ -49,8 +49,9 @@ func dropStagedMemoryImage(ctx context.Context, restoreDir string) { } // dropActorSnapshotCacheAsync pins the actor's snapshot files in restoreDir -// while the caller holds the actor lock, then evicts their clean pages from the -// host page cache in the background. +// while the caller holds the actor lock, then evicts them from the host page +// cache in the background, once they are on disk. A snapshot taken moments ago +// is still being written back. func dropActorSnapshotCacheAsync(restoreDir string, includeMemory bool) { targets := snapshotcache.FindFiles(restoreDir, func(name string) bool { return isSnapshotDataFile(name, includeMemory) @@ -58,5 +59,5 @@ func dropActorSnapshotCacheAsync(restoreDir string, includeMemory bool) { if len(targets) == 0 { return } - go snapshotcache.EvictWithRetries(targets) + go snapshotcache.SyncAndEvict(targets) } diff --git a/internal/ateworker/cgroup/actor.go b/internal/ateworker/cgroup/actor.go index c71bdce014..3d64da4e51 100644 --- a/internal/ateworker/cgroup/actor.go +++ b/internal/ateworker/cgroup/actor.go @@ -23,6 +23,7 @@ import ( "io/fs" "os" "path/filepath" + "strconv" "strings" "syscall" "time" @@ -98,6 +99,88 @@ func (l *ActorLeaf) Close() error { return l.dir.Close() } +// LimitActorLeafGrowth sets memory.high on the actor's leaf to ceiling bytes, or +// to its current usage plus headroom bytes if that is more, plus borrow bytes, +// and returns a func that puts back the memory.high it replaced. Past +// memory.high the kernel reclaims the leaf's page cache and throttles whoever +// is allocating, where at the worker pod's memory.max it may OOM-kill instead. +func LimitActorLeafGrowth(actorUID string, headroom, ceiling, borrow int64) (func(), error) { + return limitLeafGrowth(Root, actorUID, headroom, ceiling, borrow) +} + +func limitLeafGrowth(root, actorUID string, headroom, ceiling, borrow int64) (func(), error) { + current, err := actorLeafMemory(root, actorUID) + if err != nil { + return nil, err + } + path, err := actorLeafPath(root, actorUID) + if err != nil { + return nil, err + } + high := filepath.Join(path, "memory.high") + previous, err := os.ReadFile(high) + if err != nil { + return nil, fmt.Errorf("while limiting cgroup %q: %w", path, err) + } + if err := writeExisting(high, strconv.FormatInt(max(current+headroom, ceiling)+borrow, 10)); err != nil { + return nil, fmt.Errorf("while limiting cgroup %q: %w", path, err) + } + return func() { + // the leaf may already be gone, and its limit with it + _ = writeExisting(high, strings.TrimSpace(string(previous))) + }, nil +} + +// ActorLeafMemory returns the actor's leaf memory.current. +func ActorLeafMemory(actorUID string) (int64, error) { + return actorLeafMemory(Root, actorUID) +} + +func actorLeafMemory(root, actorUID string) (int64, error) { + path, err := actorLeafPath(root, actorUID) + if err != nil { + return 0, err + } + raw, err := os.ReadFile(filepath.Join(path, "memory.current")) + if err != nil { + return 0, fmt.Errorf("while reading cgroup %q memory usage: %w", path, err) + } + current, err := parseBytes(raw) + if err != nil { + return 0, fmt.Errorf("while parsing cgroup %q memory usage %q: %w", path, raw, err) + } + return current, nil +} + +// PodMemory returns the worker pod's memory limit and usage. limit is 0 when +// the pod has none. +func PodMemory() (limit, current int64, err error) { + return podMemory(Root) +} + +func podMemory(root string) (limit, current int64, err error) { + raw, err := os.ReadFile(filepath.Join(root, "memory.max")) + if err != nil { + return 0, 0, err + } + if v := strings.TrimSpace(string(raw)); v != "max" { + if limit, err = parseBytes(raw); err != nil { + return 0, 0, fmt.Errorf("while parsing memory.max %q: %w", raw, err) + } + } + if raw, err = os.ReadFile(filepath.Join(root, "memory.current")); err != nil { + return 0, 0, err + } + if current, err = parseBytes(raw); err != nil { + return 0, 0, fmt.Errorf("while parsing memory.current %q: %w", raw, err) + } + return limit, current, nil +} + +func parseBytes(raw []byte) (int64, error) { + return strconv.ParseInt(strings.TrimSpace(string(raw)), 10, 64) +} + // KillActorLeaf kills every process in the named leaf, including any they // forked, and waits until the leaf is empty or ctx is done. A missing leaf, or a // kernel without cgroup.kill, has nothing done. diff --git a/internal/ateworker/cgroup/actor_test.go b/internal/ateworker/cgroup/actor_test.go index 19aa017772..69163b85f0 100644 --- a/internal/ateworker/cgroup/actor_test.go +++ b/internal/ateworker/cgroup/actor_test.go @@ -95,3 +95,98 @@ func TestActorLeafRejectsUnsafeNames(t *testing.T) { } } } + +func TestLimitLeafGrowth(t *testing.T) { + root := t.TempDir() + // A regular directory stands in for cgroupfs, which provides both files itself. + dir := filepath.Join(root, "uid-a") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "memory.current"), []byte("1000\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "memory.high"), []byte("max\n"), 0o644); err != nil { + t.Fatal(err) + } + release, err := limitLeafGrowth(root, "uid-a", 24, 0, 0) + if err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(filepath.Join(dir, "memory.high")); string(got) != "1024" { + t.Errorf("memory.high = %q, want %q", got, "1024") + } + if _, err := limitLeafGrowth(root, "uid-a", 24, 4096, 0); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(filepath.Join(dir, "memory.high")); string(got) != "4096" { + t.Errorf("memory.high with a ceiling = %q, want %q", got, "4096") + } + if _, err := limitLeafGrowth(root, "uid-a", 24, 4096, 100); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(filepath.Join(dir, "memory.high")); string(got) != "4196" { + t.Errorf("memory.high with a borrow = %q, want %q", got, "4196") + } + release() + if got, _ := os.ReadFile(filepath.Join(dir, "memory.high")); string(got) != "max" { + t.Errorf("memory.high after release = %q, want %q", got, "max") + } + // Releasing after the leaf is gone is a no-op. + if err := os.RemoveAll(dir); err != nil { + t.Fatal(err) + } + release() +} + +// A leaf that already has a memory.high gets it back on release. +func TestLimitLeafGrowthRestoresMemoryHigh(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "uid-a") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + for name, v := range map[string]string{"memory.current": "1000\n", "memory.high": "2048\n"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(v), 0o644); err != nil { + t.Fatal(err) + } + } + release, err := limitLeafGrowth(root, "uid-a", 24, 4096, 0) + if err != nil { + t.Fatal(err) + } + release() + if got, _ := os.ReadFile(filepath.Join(dir, "memory.high")); string(got) != "2048" { + t.Errorf("memory.high after release = %q, want %q", got, "2048") + } +} + +// Without the memory controller there is no memory.high to write. +func TestLimitLeafGrowthWithoutTheMemoryController(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "uid-a"), 0o755); err != nil { + t.Fatal(err) + } + if _, err := limitLeafGrowth(root, "uid-a", 24, 0, 0); err == nil { + t.Error("limitLeafGrowth succeeded without memory.current") + } +} + +func TestPodMemory(t *testing.T) { + root := t.TempDir() + for name, content := range map[string]string{"memory.max": "4294967296\n", "memory.current": "1073741824\n"} { + if err := os.WriteFile(filepath.Join(root, name), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + limit, current, err := podMemory(root) + if err != nil || limit != 4<<30 || current != 1<<30 { + t.Errorf("podMemory() = %d, %d, %v, want %d, %d", limit, current, err, int64(4<<30), int64(1<<30)) + } + if err := os.WriteFile(filepath.Join(root, "memory.max"), []byte("max\n"), 0o644); err != nil { + t.Fatal(err) + } + if limit, _, err := podMemory(root); err != nil || limit != 0 { + t.Errorf("podMemory() without a limit = %d, %v, want 0", limit, err) + } +} diff --git a/internal/ateworker/memborrow/memborrow.go b/internal/ateworker/memborrow/memborrow.go new file mode 100644 index 0000000000..ff33863204 --- /dev/null +++ b/internal/ateworker/memborrow/memborrow.go @@ -0,0 +1,125 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package memborrow lends snapshot writes the worker pod's unbooked, free +// memory for their page cache, and holds back placements that would grow into +// it until it is written back. +package memborrow + +import ( + "context" + "fmt" + "log/slog" + "sync" + + "go.ate.dev/substrate/internal/ateworker/cgroup" +) + +// margin is pod memory a write leaves free when it borrows. +const margin = 32 << 20 + +// Ledger is memory snapshot writes hold for their page cache beyond what their +// actors booked: capacity no actor has booked, that the pod also has free. +type Ledger struct { + // Unbooked is the worker's reported memory capacity, for a pod memory + // limit, less what its hosted actors booked. + Unbooked func(podLimit int64) int64 + // PodMemory replaces cgroup.PodMemory in tests. + PodMemory func() (limit, current int64, err error) + + mu sync.Mutex + bytes int64 + // returned is closed and replaced whenever bytes drops + returned chan struct{} +} + +func (l *Ledger) podMemory() (limit, current int64, err error) { + if l.PodMemory != nil { + return l.PodMemory() + } + return cgroup.PodMemory() +} + +// Take borrows up to want. The caller passes what it got to Return once the +// pages it held are on disk. A nil Ledger lends nothing. +func (l *Ledger) Take(want int64) int64 { + if l == nil { + return 0 + } + limit, current, err := l.podMemory() + if err != nil || limit == 0 { + return 0 + } + l.mu.Lock() + defer l.mu.Unlock() + n := min(want, l.Unbooked(limit)-l.bytes, limit-current-margin) + if n <= 0 { + return 0 + } + l.bytes += n + return n +} + +// Return gives back n taken by Take. +func (l *Ledger) Return(n int64) { + if l == nil || n <= 0 { + return + } + l.mu.Lock() + defer l.mu.Unlock() + l.bytes -= n + if l.returned != nil { + close(l.returned) + } + l.returned = make(chan struct{}) +} + +// Wait holds a newly hosted actor, already counted by Unbooked, until what is +// borrowed fits in what is still unbooked, so it doesn't grow into dirty page +// cache. +func (l *Ledger) Wait(ctx context.Context) error { + if l == nil { + return nil + } + logged := false + for { + l.mu.Lock() + if l.bytes == 0 { + l.mu.Unlock() + return nil + } + limit, _, err := l.podMemory() + if err != nil || l.bytes <= l.Unbooked(limit) { + l.mu.Unlock() + return nil + } + if l.returned == nil { + l.returned = make(chan struct{}) + } + returned := l.returned + borrowed := l.bytes + l.mu.Unlock() + if !logged { + slog.InfoContext(ctx, "Waiting for snapshot writeback before booting", slog.Int64("borrowedBytes", borrowed)) + logged = true + } + select { + case <-returned: + case <-ctx.Done(): + return fmt.Errorf("while waiting for snapshot writeback: %w", ctx.Err()) + } + } +} diff --git a/internal/ateworker/memborrow/memborrow_test.go b/internal/ateworker/memborrow/memborrow_test.go new file mode 100644 index 0000000000..a3a12d10c7 --- /dev/null +++ b/internal/ateworker/memborrow/memborrow_test.go @@ -0,0 +1,109 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memborrow + +import ( + "context" + "sync/atomic" + "testing" + "time" +) + +// want is more than any test pod has to lend +const want = 1 << 40 + +func ledger(limit, current int64, unbooked *atomic.Int64) *Ledger { + return &Ledger{ + Unbooked: func(int64) int64 { return unbooked.Load() }, + PodMemory: func() (int64, int64, error) { return limit, current, nil }, + } +} + +func unbookedOf(n int64) *atomic.Int64 { + var v atomic.Int64 + v.Store(n) + return &v +} + +func TestTake(t *testing.T) { + const limit = 4 << 30 + for _, tc := range []struct { + name string + current int64 + unbooked int64 + want int64 + got int64 + }{ + {name: "capped by unbooked capacity", current: 1 << 30, unbooked: 2 << 30, want: want, got: 2 << 30}, + {name: "capped by free pod memory", current: limit - 512<<20, unbooked: 2 << 30, want: want, got: 512<<20 - margin}, + {name: "capped by what it wants", current: 0, unbooked: 2 << 30, want: 100 << 20, got: 100 << 20}, + {name: "pod full", current: limit, unbooked: 2 << 30, want: want, got: 0}, + {name: "nothing wanted", current: 0, unbooked: 2 << 30, want: -1, got: 0}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := ledger(limit, tc.current, unbookedOf(tc.unbooked)).Take(tc.want); got != tc.got { + t.Errorf("Take() = %d, want %d", got, tc.got) + } + }) + } +} + +func TestTakeSharesUnbooked(t *testing.T) { + l := ledger(4<<30, 0, unbookedOf(2<<30)) + first := l.Take(want) + if second := l.Take(want); second != 0 { + t.Errorf("second Take() = %d, want 0 with %d already borrowed", second, first) + } + l.Return(first) + if again := l.Take(want); again != first { + t.Errorf("Take() after Return = %d, want %d", again, first) + } +} + +func TestTakeWithoutPodLimit(t *testing.T) { + if got := ledger(0, 1<<30, unbookedOf(2<<30)).Take(want); got != 0 { + t.Errorf("Take() without a pod limit = %d, want 0", got) + } +} + +func TestWait(t *testing.T) { + unbooked := unbookedOf(2 << 30) + l := ledger(4<<30, 1<<30, unbooked) + borrowed := l.Take(1 << 30) + // a placement that still leaves room for the borrow goes ahead + if err := l.Wait(context.Background()); err != nil { + t.Fatal(err) + } + // one that would grow into it waits for its return + unbooked.Store(512 << 20) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if err := l.Wait(ctx); err == nil { + t.Fatal("placement into borrowed memory did not wait") + } + done := make(chan error, 1) + go func() { done <- l.Wait(context.Background()) }() + l.Return(borrowed) + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("placement still waiting after the borrow was returned") + } +} diff --git a/internal/snapshotcache/snapshotcache.go b/internal/snapshotcache/snapshotcache.go index a1e2c710db..c3812a96b7 100644 --- a/internal/snapshotcache/snapshotcache.go +++ b/internal/snapshotcache/snapshotcache.go @@ -111,6 +111,45 @@ func DropFiles(files []File) bool { return dropped } +// StartWriteback writes each path's dirty pages to disk in the background and +// then calls done, if set. Writing can block on a busy disk, and callers don't +// wait. +func StartWriteback(paths []string, done func()) error { + var files []*os.File + for _, p := range paths { + f, err := os.Open(p) + if err != nil { + for _, f := range files { + _ = f.Close() + } + return err + } + files = append(files, f) + } + go func() { + for _, f := range files { + _ = unix.Fdatasync(int(f.Fd())) + _ = f.Close() + } + if done != nil { + done() + } + }() + return nil +} + +// SyncAndEvict waits for targets to reach disk and then evicts them, so pages +// still dirty from a recent snapshot get dropped too. +func SyncAndEvict(targets []File) { + for _, t := range targets { + if f, err := os.Open(t.Path); err == nil { + _ = unix.Fdatasync(int(f.Fd())) + _ = f.Close() + } + } + EvictWithRetries(targets) +} + // EvictFiles evicts clean cached pages of targets via POSIX_FADV_DONTNEED and // returns true if any target still has dirty or writeback pages remaining in // cache (or if fadvise failed). Files with dirty or writeback pages are skipped diff --git a/internal/tarutil/tarutil.go b/internal/tarutil/tarutil.go index 9c0150a3b4..436ec33110 100644 --- a/internal/tarutil/tarutil.go +++ b/internal/tarutil/tarutil.go @@ -82,6 +82,34 @@ func copyPooled(dst io.Writer, src io.Reader) (int64, error) { return io.CopyBuffer(writerOnly{dst}, readerOnly{src}, *bp) } +// writeBehindChunk is how much of an archive may be dirty in the page cache +// before we write it out and drop it. Dirty pages can't be reclaimed, and +// piling up a whole archive of them can run a worker pod out of memory. +const writeBehindChunk = 8 << 20 + +// writeBehind writes to f, writing out and dropping each full chunk. +type writeBehind struct { + f *os.File + written int64 + flushed int64 +} + +func (w *writeBehind) Write(p []byte) (int, error) { + n, err := w.f.Write(p) + w.written += int64(n) + if err != nil || w.written-w.flushed < writeBehindChunk { + return n, err + } + fd, off, size := int(w.f.Fd()), w.flushed, w.written-w.flushed + if err := unix.SyncFileRange(fd, off, size, unix.SYNC_FILE_RANGE_WAIT_BEFORE|unix.SYNC_FILE_RANGE_WRITE|unix.SYNC_FILE_RANGE_WAIT_AFTER); err != nil { + return n, fmt.Errorf("while writing out archive: %w", err) + } + // best effort, the range is on disk already + _ = unix.Fadvise(fd, off, size, unix.FADV_DONTNEED) + w.flushed = w.written + return n, nil +} + type writerOnly struct{ io.Writer } type readerOnly struct{ io.Reader } @@ -129,7 +157,7 @@ func create(ctx context.Context, tarPath, srcDir string, skip SkipFunc, includeR defer f.Close() bw := tarWriterPool.Get().(*bufio.Writer) - bw.Reset(f) + bw.Reset(&writeBehind{f: f}) defer func() { bw.Reset(nil) tarWriterPool.Put(bw) diff --git a/internal/tarutil/tarutil_test.go b/internal/tarutil/tarutil_test.go index 47572ae180..9b9a660260 100644 --- a/internal/tarutil/tarutil_test.go +++ b/internal/tarutil/tarutil_test.go @@ -18,6 +18,7 @@ package tarutil import ( "archive/tar" + "bytes" "context" "errors" "net" @@ -191,6 +192,33 @@ func TestRoundTrip(t *testing.T) { }) } +// An archive bigger than writeBehindChunk is written out a chunk at a time. +func TestRoundTripLargeFile(t *testing.T) { + src := t.TempDir() + want := make([]byte, 2*writeBehindChunk+12345) + for i := range want { + want[i] = byte(i % 251) + } + if err := os.WriteFile(filepath.Join(src, "big"), want, 0o644); err != nil { + t.Fatal(err) + } + tarPath := filepath.Join(t.TempDir(), "out.tar") + if err := Create(context.Background(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + got, err := os.ReadFile(filepath.Join(dst, "big")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Errorf("extracted %d bytes that differ from the %d archived", len(got), len(want)) + } +} + func TestCreateEmptyDir(t *testing.T) { tarPath := filepath.Join(t.TempDir(), "empty.tar") if err := Create(t.Context(), tarPath, t.TempDir()); err != nil { From cae252a5ecc9ff6e99842274ed130627be41168a Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Wed, 28 Oct 2026 19:42:01 +0000 Subject: [PATCH 38/53] AGENTS.md: add the version skew rules (#3151) List the version skew rules a code change is most likely to break, so developers' agents follow them without opening the policy. They apply from v1.0.0. - [x] Tests pass - [x] Docs updated --------- Signed-off-by: Haven Xia Co-authored-by: Davanum Srinivas --- AGENTS.md | 4 ++++ docs/version-skew-policy.md | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c393aafa88..f1db3e0fdf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,6 +130,10 @@ Agent Substrate makes no compatibility guarantee before v1.0.0. Do not add code Remove or rename fields, RPCs, flags, and types outright and update every caller in the same change. Do not add `reserved` statements to protos, deprecated aliases, fallbacks for old formats, or version negotiation. +## Version Skew + +From v1.0.0, changes must keep upgrades and rollbacks working under [docs/version-skew-policy.md](docs/version-skew-policy.md). Read it before you change an RPC, a CRD, the worker pod template, a database migration, or file(s) that `atelet` and `ateworker` share on a node. + ## Commit Messages - **Describe the change and why it was needed**: The message is read on `main` long after the PR branch is gone, so it should stand on its own. diff --git a/docs/version-skew-policy.md b/docs/version-skew-policy.md index 3346f19e1d..64427c1b7a 100644 --- a/docs/version-skew-policy.md +++ b/docs/version-skew-policy.md @@ -72,7 +72,7 @@ Clients: Wherever a caller can be newer than its server, the server learns a new field in one minor release and the caller sends it only from the next minor release on (two-step releases). ### CRDs -The N+1 CRDs are applied with `ate-controller` N+1. Components that upgrade before it read the N CRDs, so users must not set new CRD fields until the upgrade completes. +The N+1 CRDs are applied with `ate-controller` N+1. Components that upgrade before it run at N+1 against the N CRDs. `atelet`, for one, watches WorkerPool and SandboxConfig. So these components must work without a new CRD field or kind. Users must not set new CRD fields until the upgrade completes. Between compatible releases, a CRD change only adds optional fields. The CRDs serve one version with no conversion. So a rollback from N+1 that applies the N CRDs again removes every field N does not know from every object. @@ -99,7 +99,7 @@ This section is about data one release writes and another release reads, whether * Data on the node: the directory layout and the files `atelet` keeps there. * What N writes, N+1 can read. What N+1 writes, N can read. * So a layout change takes two minor releases: the first reads the old and the new layout but still writes the old one, and the next writes the new one. This keeps a rollback possible. Neither step can go in a patch release: an upgrade can skip patch releases, and a rollback can land on any of them. - * `atelet` passes the per-actor directories to `ateworker` in the RPC, so only `atelet` knows the layout. The sockets `atelet` and `ateworker` use to reach each other do not change within a major. + * `atelet` passes the per-actor directories to `ateworker` in the RPC. `ateworker` also reads some files `atelet` writes under the paths in `internal/nodepath`, such as `sandbox-runtimes.json`, and rejects fields it does not know in them. So a new field in those files follows the two-step rule of the RPC: `ateworker` learns it in one minor release, and `atelet` writes it from the next. The sockets `atelet` and `ateworker` use to reach each other do not change within a major. * Data in snapshots: `manifest.json`, which `atelet` writes and reads. * Within a major, a release reads the snapshot data any earlier release of that major wrote. * A new field must be safe to ignore. A release ignores manifest fields it does not know, so a snapshot taken at N+1 still restores after a rollback to N. From 475f49b023352b47b8504ffead596953f28d7e7f Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Wed, 28 Oct 2026 19:50:31 +0000 Subject: [PATCH 39/53] ateapi: carry the golden actor's crash message into the template status (#2644) When the actor used to build a template crashes, the template status now includes the actor's crash message. This makes failures such as a bad image volume or failed restore visible directly on the template. Overlaps with #2453, which also adds crash details to template failures. - [x] Tests pass: `go test -race ./cmd/ateapi/internal/controlapi/`; `go vet`. - [x] Appropriate changes to documentation are included in the PR: none needed. Signed-off-by: Davanum Srinivas --- cmd/ateapi/internal/controlapi/template_reconciler.go | 6 +++++- .../internal/controlapi/template_reconciler_test.go | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 2f0c979145..a2e5395956 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -259,7 +259,11 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource switch state := actor.GetStatus().GetState(); state { case ateapi.ActorState_ACTOR_STATE_CRASHED: - return 0, r.fail(ctx, tmpl, reasonGoldenActorCrashed, "golden actor crashed before its snapshot was taken") + msg := "golden actor crashed before its snapshot was taken" + if crash := actor.GetStatus().GetCrash().GetMessage(); crash != "" { + msg += ": " + crash + } + return 0, r.fail(ctx, tmpl, reasonGoldenActorCrashed, msg) case ateapi.ActorState_ACTOR_STATE_RUNNING: takeAt := goldenSnapshotStatus.GetSnapshotTime() diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index c2eea80cc0..5abd08e491 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -166,6 +166,7 @@ type fakeGoldenControl struct { exists bool goldenState ateapi.ActorState goldenSnapshot string + goldenCrash string // snapshot is the external snapshot a completed suspend produces; empty // simulates a suspend that wrote none. snapshot string @@ -185,6 +186,9 @@ func (c *fakeGoldenControl) CreateAtespace(_ context.Context, req *ateapi.Create func (c *fakeGoldenControl) goldenActorStatus() *ateapi.ActorStatus { st := &ateapi.ActorStatus{State: c.goldenState} + if c.goldenCrash != "" { + st.Crash = &ateapi.ActorCrash{Message: c.goldenCrash} + } if c.goldenSnapshot != "" { st.LastAssignedGeneration = 1 st.Snapshots = []*ateapi.Snapshot{ @@ -534,9 +538,9 @@ func TestReconcileOne(t *testing.T) { { name: "crashed golden actor fails the template", template: testTemplate(), - control: &fakeGoldenControl{exists: true, goldenState: ateapi.ActorState_ACTOR_STATE_CRASHED}, + control: &fakeGoldenControl{exists: true, goldenState: ateapi.ActorState_ACTOR_STATE_CRASHED, goldenCrash: "atelet Run: unhandled tar entry"}, wantFailedReason: reasonGoldenActorCrashed, - wantMessage: "crashed", + wantMessage: "crashed before its snapshot was taken: atelet Run: unhandled tar entry", }, { name: "resume failure requeues without failing", From 2f1be4b2e324caa0a30793f339bad55175aef1b0 Mon Sep 17 00:00:00 2001 From: Sairaj Pokale Date: Wed, 28 Oct 2026 21:47:10 +0000 Subject: [PATCH 40/53] benchmarking: remove default ATE_ENV_IMAGE and stop deploying ate-env template by default (#3366) Remove the default `ATE_ENV_IMAGE` digest and exclude `ate-env` from the default `WORKLOAD_TEMPLATES` list. `deploy.sh` now deploys `ate-env` only when `ATE_ENV_IMAGE` is set (or `WORKLOAD_TEMPLATES` includes `ate-env`), and always includes it on `--delete` so teardown cleans up the template. - [x] Tests pass - [x] Docs updated Signed-off-by: Sairaj Pokale --- benchmarking/README.md | 17 +++++++++++------ benchmarking/workloads/deploy.sh | 17 ++++++++++++----- .../manifests/ate-env-template.yaml.tmpl | 4 +--- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/benchmarking/README.md b/benchmarking/README.md index 2f315fb39d..402219a40f 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -273,17 +273,22 @@ nothing. Durations are in milliseconds. ### ate-env Benchmark The `ate-env` benchmark (`--user-class ate_env`) exercises the -[`ate-env-guest`](https://github.com/agent-substrate/env/tree/main/cmd/ate-env-guest) -sandbox image from `agent-substrate/env`. `workloads/deploy.sh` deploys and -deletes the `ate-env` `ActorTemplate` by default using the pinned release guest -image (see -[`workloads/manifests/ate-env-template.yaml.tmpl`](workloads/manifests/ate-env-template.yaml.tmpl) -to override the guest image with a custom build). It follows `glutton`'s +[`ate-env-guest`](https://github.com/agent-substrate/env/tree/1c08a7a578afd896373511a83beb242aec1425c0/cmd/ate-env-guest) +sandbox image from `agent-substrate/env`. It follows `glutton`'s wake/hold/hibernate lifecycle, but replaces synthetic HTTP `/ping` calls with a real `"hello world"` process execution inside the guest (`StartProcess(["echo", "hello world"])` + `StreamProcessOutput(follow: true)`) over gRPC directly through `atenet-router` (`1 dns:///` gRPC channel per VU). +Build the guest image from `agent-substrate/env` at commit `1c08a7a` (see +[`workloads/manifests/ate-env-template.yaml.tmpl`](workloads/manifests/ate-env-template.yaml.tmpl)), +then deploy the `ate-env` template and user class: + +```bash +ATE_ENV_IMAGE= WORKLOAD_TEMPLATES=ate-env ./benchmarking/workloads/deploy.sh --deploy +./benchmarking/locust/deploy.sh --deploy --user-class ate_env +``` + #### ate-env Configuration Knobs Reuses `glutton`'s runtime and startup knobs: diff --git a/benchmarking/workloads/deploy.sh b/benchmarking/workloads/deploy.sh index 010a2cc295..7881ca2820 100755 --- a/benchmarking/workloads/deploy.sh +++ b/benchmarking/workloads/deploy.sh @@ -39,8 +39,8 @@ SWEPERF_TEMPLATE_DIR="${MANIFEST_DIR}/sweperf/generated" # overrides the default set — the usermem and kernelmem templates (for the # matching locust tests) are not deployed by default. The glutton-storage # template is added when a StorageClass is set, because it needs a CSI driver -# that not every cluster has. -read -r -a TEMPLATES <<<"${WORKLOAD_TEMPLATES:-sleep glutton glutton-durdir-data glutton-durdir-full ate-env}" +# that not every cluster has; ate-env is added when ATE_ENV_IMAGE is set. +read -r -a TEMPLATES <<<"${WORKLOAD_TEMPLATES:-sleep glutton glutton-durdir-data glutton-durdir-full}" # template_manifest prints the manifest path for ActorTemplate NAME. template_manifest() { @@ -72,9 +72,9 @@ WORKER_MEMORY="" # The StorageClass for the glutton-storage external volume. Empty means the # template is not deployed. STORAGE_CLASS_NAME="${STORAGE_CLASS_NAME:-}" -# The image for the ate-env benchmark ActorTemplate. Defaults to the pinned -# release guest image; set ATE_ENV_IMAGE to override with a custom build. -ATE_ENV_IMAGE="${ATE_ENV_IMAGE:-us-docker.pkg.dev/gke-substrate-release/benchmark/ate-env-guest@sha256:1d38934ef1dac71fcb1f1286de734e06ab2e19747279bccbd37bfe69e0d5bc72}" +# The image for the ate-env benchmark ActorTemplate. Empty means the +# template is not deployed. +ATE_ENV_IMAGE="${ATE_ENV_IMAGE:-}" # The address to which an instrumented actor container sends its telemetry. # --otlp-endpoint sets it. Without the flag, resolve_otlp_endpoint reads the # address that the control plane uses. @@ -475,6 +475,13 @@ if [[ -z "${WORKLOAD_TEMPLATES:-}" ]] \ TEMPLATES+=(glutton-storage) fi +# ate-env joins the default set only when ATE_ENV_IMAGE is set. Delete always +# includes it, because deleting a template that does not exist is a no-op. +if [[ -z "${WORKLOAD_TEMPLATES:-}" ]] \ + && { [[ -n "${ATE_ENV_IMAGE}" ]] || [[ "${action}" == "delete" ]]; }; then + TEMPLATES+=(ate-env) +fi + if [[ "${action}" == "deploy" && -z "${STORAGE_CLASS_NAME}" && " ${TEMPLATES[*]} " == *" glutton-storage "* ]]; then echo "Error: the glutton-storage template needs --storage-class-name" >&2 exit 1 diff --git a/benchmarking/workloads/manifests/ate-env-template.yaml.tmpl b/benchmarking/workloads/manifests/ate-env-template.yaml.tmpl index 2640d0846e..64d6791643 100644 --- a/benchmarking/workloads/manifests/ate-env-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/ate-env-template.yaml.tmpl @@ -29,9 +29,7 @@ containers: # # KO_DOCKER_REPO="gcr.io//ate-env-guest" ko build --bare --platform=linux/amd64 ./cmd/ate-env-guest # - # Substituted by benchmarking/workloads/deploy.sh (which defaults ATE_ENV_IMAGE - # to us-docker.pkg.dev/gke-substrate-release/benchmark/ate-env-guest@sha256:1d38934ef1dac71fcb1f1286de734e06ab2e19747279bccbd37bfe69e0d5bc72; - # set ATE_ENV_IMAGE to override with a custom digest). + # Substituted from ATE_ENV_IMAGE by benchmarking/workloads/deploy.sh. image: "${ATE_ENV_IMAGE}" command: - "/ko-app/ate-env-guest" From e641e6ba34be18c5800f46c549db0a67ec83650c Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Wed, 28 Oct 2026 21:48:21 +0000 Subject: [PATCH 41/53] Delete hack/install-ate.sh and hack/install-ate-kind.sh (#3354) Since #2718, both scripts only forward to `go run ./cmd/ate-setup`. This PR deletes them, and every caller, CI step and doc now runs `ate-setup` directly. CI passes `--kind` in place of `install-ate-kind.sh`. The kind Quickstart sets the same option once with `export ATE_INSTALL_KIND=true`, so the demo READMEs no longer mention kind or GKE. Fix #1503 ### Breaking Change There are some behavior changes: - `ate-setup` exits 1 on an unknown subcommand such as `deploy demo demo-counter`, instead of printing help and exiting 0, this fixes #3209 - `--setup-csi` no longer accepts the old shell spellings `true`, `false` or an empty value because they are returned `unknown CSI driver`, the previous params are left for compatiblity with shell script. - `hack/setup-csi-hostpath-kind.sh` is replaced by `ate-setup --kind setup csi hostpath`. ### Testing Ran every demo README as written on fresh kind and GKE clusters. Every `ate-setup` step works on both. claude-code-multiplex, parking and the counter microVM script still fail, as they did before this PR (#2487, #3287; #2092 rewrites the microVM script). Claude help review the list and do the replacement, I reviewed. - [x] Tests pass - [x] Docs updated --------- Signed-off-by: Haven Xia --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/pr-workflow.yaml | 37 +--- AGENTS.md | 4 +- README.md | 12 +- benchmarking/README.md | 8 +- benchmarking/automation/README.md | 4 +- benchmarking/automation/orchestrator.py | 10 +- benchmarking/automation/test_orchestrator.py | 4 +- benchmarking/automation/tests.yaml | 4 +- benchmarking/automation/util.py | 2 +- benchmarking/nighthawk-egress/README.md | 6 +- benchmarking/nighthawk-ingress/README.md | 4 +- benchmarking/nighthawk-ingress/run-dev.sh | 2 +- benchmarking/telemetry/README.md | 2 +- cmd/ate-setup/README.md | 9 +- cmd/ate-setup/cli-diff.md | 4 - cmd/ate-setup/commands.md | 192 ++++++++---------- cmd/ate-setup/differences.md | 26 +-- cmd/ate-setup/internal/cmd/csi_test.go | 2 +- cmd/ate-setup/internal/cmd/root.go | 10 +- cmd/ate-setup/internal/config/config.go | 7 +- cmd/ate-setup/internal/config/config_test.go | 9 +- cmd/ate-setup/internal/config/file.go | 3 +- cmd/ate-setup/internal/config/load.go | 17 +- cmd/ate-setup/internal/config/load_test.go | 10 +- cmd/ate-setup/internal/steps/apienv.go | 2 +- cmd/ate-setup/internal/steps/csi.go | 7 +- cmd/ate-setup/internal/steps/csi_test.go | 11 +- cmd/ate-setup/main.go | 5 +- cmd/ate-setup/preflight_test.go | 32 +-- cmd/kubectl-ate/README.md | 2 +- .../internal/cmd/actortemplate_test.go | 2 +- demos/claude-code-multiplex/README.md | 11 +- demos/counter/README.md | 19 +- demos/counter/counter-template.yaml.tmpl | 2 +- demos/egress/README.md | 27 +-- demos/egress/test-egress.sh | 9 +- demos/jupyter/README.md | 38 +--- demos/multi-template/README.md | 8 +- demos/parking/README.md | 8 +- demos/sandbox/README.md | 8 +- docs/authorization.md | 6 +- docs/dev/best-practices/otel-collector.md | 4 +- docs/dev/mitm-egress-test-runbook.md | 6 +- docs/egress-credential-injection.md | 4 +- docs/examples/substrate-install.yaml | 2 +- docs/operator-install.md | 6 +- hack/install-ate-kind.sh | 44 ---- hack/install-ate.sh | 35 ---- hack/install-microvm-deps.sh | 4 +- hack/microvm-assets/README.md | 5 +- hack/microvm-assets/stage-to-rustfs.sh | 2 +- hack/run-e2e-kind.sh | 6 +- hack/run-microvm-demo-kind.sh | 4 +- hack/run-microvm-demo.sh | 33 ++- hack/setup-csi-hostpath-kind.sh | 140 ------------- .../deploy/example/storageclass-nfs.yaml | 4 +- hack/verify-atenet-drain.sh | 3 +- hack/verify-egress-demo.sh | 2 +- internal/e2e/README.md | 10 +- internal/e2e/credprovider.go | 2 +- internal/e2e/env.go | 4 +- internal/e2e/sandbox.go | 12 +- internal/e2e/sandbox_test.go | 4 +- .../egresscredinject/egresscredinject_test.go | 2 +- .../e2e/suites/egressmitm/goldenboot_test.go | 2 +- internal/e2e/suites/security/README.md | 2 +- internal/e2e/trustbundle.go | 2 +- manifests/ate-install/atenet-egress.yaml | 2 +- manifests/ate-install/base/kustomization.yaml | 15 +- .../cloudsql/proxy-sidecar-patch.yaml | 4 +- .../kind/atelet/kustomization.yaml | 2 +- manifests/ate-install/kind/kustomization.yaml | 7 +- .../podcert-size10/kustomization.yaml | 2 +- .../postgres-config-patch.yaml | 7 +- .../microvm/sandboxconfig-microvm.yaml.tmpl | 2 +- tools/setup-gcp/README.md | 2 +- tools/setup-gcp/cloud-sql.md | 4 +- tools/setup-gcp/cmd/cloudsql.go | 6 +- tools/setup-gcp/cmd/cloudsql_test.go | 2 +- 80 files changed, 317 insertions(+), 669 deletions(-) delete mode 100644 cmd/ate-setup/cli-diff.md delete mode 100755 hack/install-ate-kind.sh delete mode 100755 hack/install-ate.sh delete mode 100755 hack/setup-csi-hostpath-kind.sh diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index b320dd3366..26879fe159 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -47,7 +47,7 @@ body: label: Steps to Reproduce description: How can the issue be reproduced? Provide commands, scripts, or manifests (e.g., ActorTemplate). placeholder: | - 1. Deploy Substrate with `hack/install-ate-kind.sh ...` + 1. Deploy Substrate with `go run ./cmd/ate-setup --kind ...` 2. Apply manifest `kubectl apply -f ...` 3. Run command `...` 4. See error diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index a6a0b009c1..70c1e94b77 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -267,27 +267,14 @@ jobs: # gateway. ATE_PREVIEW above selects the preview features. env: ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} - NO_DEV_ENV: "true" - KO_DOCKER_REPO: "localhost:5001" - ATE_INSTALL_KIND: "true" - BUCKET_NAME: "ate-snapshot" - KIND_CLUSTER_NAME: "kind" - KUBECTL_CONTEXT: "kind-kind" - run: go run ./cmd/ate-setup deploy ate-system + run: go run ./cmd/ate-setup --kind deploy ate-system - name: Enable NFS # Load NFS kernel modules so in-cluster NFS server and CSI driver can run. run: | sudo modprobe nfs || true sudo modprobe nfsd || true - name: Install CSI NFS driver - env: - NO_DEV_ENV: "true" - KO_DOCKER_REPO: "localhost:5001" - ATE_INSTALL_KIND: "true" - BUCKET_NAME: "ate-snapshot" - KIND_CLUSTER_NAME: "kind" - KUBECTL_CONTEXT: "kind-kind" - run: go run ./cmd/ate-setup setup csi nfs + run: go run ./cmd/ate-setup --kind setup csi nfs - name: Deploy micro-VM counter demo # Stages the (cached) assets into the cluster's rustfs and deploys the # counter-microvm demo onto the control plane installed above. The demo @@ -296,29 +283,15 @@ jobs: ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} run: hack/run-microvm-demo-kind.sh - name: Deploy gVisor counter demo - env: - NO_DEV_ENV: "true" - KO_DOCKER_REPO: "localhost:5001" - ATE_INSTALL_KIND: "true" - BUCKET_NAME: "ate-snapshot" - KIND_CLUSTER_NAME: "kind" - KUBECTL_CONTEXT: "kind-kind" - run: go run ./cmd/ate-setup deploy demo counter + run: go run ./cmd/ate-setup --kind deploy demo counter - name: Deploy egress demos # TestActorEgress in the networking suite builds its Actor from the egress # ActorTemplate for the class under test, so both fixtures have to exist # before their respective lane runs. The demo's actors trust the CA the # gateway mints its leaves from. - env: - NO_DEV_ENV: "true" - KO_DOCKER_REPO: "localhost:5001" - ATE_INSTALL_KIND: "true" - BUCKET_NAME: "ate-snapshot" - KIND_CLUSTER_NAME: "kind" - KUBECTL_CONTEXT: "kind-kind" run: | - go run ./cmd/ate-setup deploy demo egress - go run ./cmd/ate-setup deploy demo egress-microvm + go run ./cmd/ate-setup --kind deploy demo egress + go run ./cmd/ate-setup --kind deploy demo egress-microvm - name: Trust local TLS origins in egress gateway run: hack/setup-e2e-egress-tls-kind.sh # scale the idle sandbox class's demo pools to zero so each lane has the node diff --git a/AGENTS.md b/AGENTS.md index f1db3e0fdf..30b87bdd71 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,7 +66,7 @@ It takes the Kubernetes control-plane out of the critical path to achieve lower Agent Substrate relies on the fact that agent-like applications tend to be idle most of the time to achieve heavy multiplexing. For development, it's recommended to read the `README.md` and `CONTRIBUTING.md` in the root folder. -See `hack/install-ate.sh` and `tools/setup-gcp` for provisioning and deploying clusters and GCP resources. +See `cmd/ate-setup` and `tools/setup-gcp` for provisioning and deploying clusters and GCP resources. ## Repository Layout @@ -104,7 +104,7 @@ See `docs/dev/code-layout.md` for the full rationale and per-directory details. The dev entry points are the scripts under `hack/` and the `cmd/ate-setup` installer. ### Building and Installing -- **Install from source**: `go run ./cmd/ate-setup deploy ate-system` (or the `hack/install-ate.sh` shim) builds every image with ko as part of the install; see `cmd/ate-setup/README.md` +- **Install from source**: `go run ./cmd/ate-setup deploy ate-system` builds every image with ko as part of the install; see `cmd/ate-setup/README.md` - **Binaries**: `go build -o bin/kubectl-ate ./cmd/kubectl-ate`, `go build -o bin/ate-setup ./cmd/ate-setup` - **Release images**: `VERSION=TAG go run ./cmd/ate-setup publish release-images --ko-docker-repo REPO` (the release set, `images.ReleaseComponents`, all tagged `TAG`; see `docs/dev/releasing.md`) - **Release binaries**: `VERSION=TAG go run ./cmd/ate-setup publish release-binaries --out DIR` (`kubectl-ate` for linux and darwin, amd64 and arm64, with `.sha256` files) diff --git a/README.md b/README.md index 253d7c47b0..6e89e288e6 100644 --- a/README.md +++ b/README.md @@ -118,12 +118,16 @@ To quickly set up the complete environment: # create cluster and local registry hack/create-kind-cluster.sh +# make ate-setup target the kind cluster; set this in every shell that runs +# ate-setup, including for the demo walkthroughs +export ATE_INSTALL_KIND=true + # install Substrate -hack/install-ate-kind.sh deploy ate-system --credential-provider='{"name":"k8s.io"}' \ +go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}' \ --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-system/sa/atenet-client # install counter demo -hack/install-ate-kind.sh deploy demo counter +go run ./cmd/ate-setup deploy demo counter # install kubectl-ate go install ./cmd/kubectl-ate @@ -215,8 +219,8 @@ go run ./tools/setup-gcp create bucket To run the PostgreSQL store backend on Cloud SQL — with IAM database authentication and no passwords — see [tools/setup-gcp/cloud-sql.md](tools/setup-gcp/cloud-sql.md). -Similarly, you can deploy or cleanup specific Agent Substrate components using -the installation script. See `go run ./cmd/ate-setup` for all options. +Similarly, you can deploy or clean up specific Agent Substrate components with +`ate-setup`. See `go run ./cmd/ate-setup --help` for all options. ```bash # Re-deploy only ate-apiserver of the ATE system go run ./cmd/ate-setup deploy apiserver diff --git a/benchmarking/README.md b/benchmarking/README.md index 402219a40f..6e9b0daf41 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -30,7 +30,7 @@ scenario ladder, read [observability.md](observability.md). > ```bash > ATE_API_AUTHZ_BOOTSTRAP_OWNERS=system:serviceaccount:ate-system:ate-client,spiffe://cluster.local/ns/benchmarking/sa/default \ > ATE_INGRESS_ALLOWED_SPIFFE_IDS=spiffe://cluster.local/ns/benchmarking/sa/default \ -> ./hack/install-ate.sh deploy ate-system +> go run ./cmd/ate-setup deploy ate-system > ``` > The nightly automation sets both itself; see > [automation/README.md](automation/README.md). @@ -154,7 +154,7 @@ and state restoration latency when a durable directory is attached to the actor. * `glutton-storage`: Attaches an external CSI volume instead of a durable directory. `deploy_locust.sh` and `workloads/deploy.sh` deploy it only when `--storage-class-name` names an existing StorageClass. To run it on - NFS, install with `hack/install-ate.sh setup csi nfs` and pass + NFS, install with `go run ./cmd/ate-setup setup csi nfs` and pass `--storage-class-name csi-nfs-sc`, the class the e2e tests also use. #### DurDir Reported Metrics @@ -633,7 +633,7 @@ limits of ate-api-server and Postgres. Isolate mode takes the nodes out by installing substrate with a fake data plane: ```bash -hack/install-ate.sh deploy ate-system --fake-data-plane +go run ./cmd/ate-setup deploy ate-system --fake-data-plane ./benchmarking/workloads/deploy.sh --deploy --worker-count 200 \ --fake-atelet-env FAKE_ATELET_DELAY=500ms ``` @@ -679,7 +679,7 @@ Fake actors need neither. fake-atelet takes the storage settings of atelet's objectstore-plugin sidecar, whose writes it takes over, so its snapshot placeholders go to the storage the install points atelet at. The install refuses a cluster whose WorkerPools still have worker pods, which would -outlive ate-controller. A normal `hack/install-ate.sh deploy ate-system` +outlive ate-controller. A normal `ate-setup deploy ate-system` puts the real data plane back, and `delete ate-system` removes the fakes either way. Never use the mode on a cluster that serves real actors: actors placed on fake Workers do not exist. diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 7f36ddf5be..50c5cd5d62 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -18,7 +18,7 @@ the router capacity benchmark — see 4. `docker build && docker push` builds the runner image for each test type in use, tagged with the commit hash: `${KO_DOCKER_REPO}/locust-test:` and/or `${KO_DOCKER_REPO}/nighthawk-ingress-test:`. -5. `hack/install-ate.sh deploy ate-system` (with `ATE_CREDENTIAL_PROVIDER` +5. `go run ./cmd/ate-setup deploy ate-system` (with `ATE_CREDENTIAL_PROVIDER` in `.ate-dev-env.sh`, or `--credential-provider` in the test's `ateArgs`) + `benchmarking/workloads/deploy.sh --deploy --sandbox-class ` (these build & push substrate / workload @@ -147,7 +147,7 @@ principal://iam.googleapis.com/projects//locations/global/worklo - `roles/container.admin` on the test cluster's project — required to manage cluster-scoped resources (CRDs, ClusterRoles, ClusterRoleBindings, - Namespaces) that `hack/install-ate.sh deploy ate-system` creates. + Namespaces) that `ate-setup deploy ate-system` creates. `container.developer` is not enough — it intentionally omits the `container.clusterRoles.*` and `container.customResourceDefinitions.*` permissions. diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index f60e90e455..c50fdc2946 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -126,7 +126,7 @@ def apply_target_cluster( target_cluster: str, target_cluster_dir: str = TARGET_CLUSTER_DIR ) -> None: """Copy target_cluster_dir/.sh into the cloned - substrate repo as .ate-dev-env.sh (so install-ate.sh / deploy.sh source + substrate repo as .ate-dev-env.sh (so ate-setup / deploy.sh source it) and merge it into this process's env (so the orchestrator's own gcloud / docker / kubectl calls see the same values). Resets os.environ to the startup baseline first so vars defined by a previous target @@ -390,7 +390,7 @@ def validate_and_normalize_tests(tests: list[dict[str, Any]]) -> None: def substrate_args(test: dict[str, Any]) -> list[str]: - """Return the hack/install-ate.sh deploy ate-system arguments for a + """Return the ate-setup deploy ate-system arguments for a tests.yaml entry: its ateArgs, then the fake data plane flag its isolate block asks for. The teardown needs no matching flag: delete ate-system removes the fake data plane whether or not it was installed.""" @@ -413,11 +413,11 @@ def deploy_substrate(ate_args: Iterable[str] = ()) -> None: "ATE_API_AUTHZ_BOOTSTRAP_OWNERS": f"{CLIENT_PRINCIPAL},{BENCHMARK_RUNNER_PRINCIPAL}", "ATE_INGRESS_ALLOWED_SPIFFE_IDS": BENCHMARK_RUNNER_PRINCIPAL, } - run(["hack/install-ate.sh", "deploy", "ate-system", *(str(a) for a in ate_args)], env=env) + run(["go", "run", "./cmd/ate-setup", "deploy", "ate-system", *(str(a) for a in ate_args)], env=env) def teardown_substrate() -> None: - run_no_check(["hack/install-ate.sh", "delete", "ate-system"]) + run_no_check(["go", "run", "./cmd/ate-setup", "delete", "ate-system"]) def install_microvm_deps() -> None: @@ -595,7 +595,7 @@ def main() -> None: # Per-target-cluster caches: re-running setup for the same target # cluster is wasted work, so we track what was last set up and only # redo it when the target cluster name changes (substrate images get - # rebuilt by install-ate.sh each test anyway via ko apply). Runner + # rebuilt by ate-setup each test anyway via ko). Runner # images are built lazily per test type, so a locust-only tests.yaml # never builds the nighthawk image and vice versa. last_target = None diff --git a/benchmarking/automation/test_orchestrator.py b/benchmarking/automation/test_orchestrator.py index 77833d43ba..3cf8f809b8 100644 --- a/benchmarking/automation/test_orchestrator.py +++ b/benchmarking/automation/test_orchestrator.py @@ -68,7 +68,7 @@ class DeploySubstrateTest(unittest.TestCase): def test_runner_and_client_are_bootstrap_owners(self, run): orchestrator.deploy_substrate(["--credential-provider={}"]) cmd = run.call_args.args[0] - self.assertEqual(cmd, ["hack/install-ate.sh", "deploy", "ate-system", "--credential-provider={}"]) + self.assertEqual(cmd, ["go", "run", "./cmd/ate-setup", "deploy", "ate-system", "--credential-provider={}"]) env = run.call_args.kwargs["env"] owners = env["ATE_API_AUTHZ_BOOTSTRAP_OWNERS"].split(",") self.assertEqual( @@ -92,7 +92,7 @@ def test_keeps_the_rest_of_the_environment(self, run): @mock.patch("orchestrator.run_no_check") def test_teardown_substrate(self, run_no_check): orchestrator.teardown_substrate() - run_no_check.assert_called_once_with(["hack/install-ate.sh", "delete", "ate-system"]) + run_no_check.assert_called_once_with(["go", "run", "./cmd/ate-setup", "delete", "ate-system"]) class DeployWorkloadsTest(unittest.TestCase): diff --git a/benchmarking/automation/tests.yaml b/benchmarking/automation/tests.yaml index a9d43827b7..1a5eae67ef 100644 --- a/benchmarking/automation/tests.yaml +++ b/benchmarking/automation/tests.yaml @@ -52,14 +52,14 @@ # large workerCount fleets whose image pulls or scheduling # outrun the default. # ateArgs Optional. List of strings appended verbatim to -# `hack/install-ate.sh deploy ate-system` when this +# `ate-setup deploy ate-system` when this # test's substrate is deployed (e.g. ["--atenet-dataplane=agentgateway"]). # The deploy requires --credential-provider; set it here # as one YAML string, '--credential-provider={"name":"k8s.io"}' # (no shell quoting: the args are passed without a shell), # or as ATE_CREDENTIAL_PROVIDER in .ate-dev-env.sh. # isolate Optional. Isolate mode: installs substrate with the fake -# data plane (hack/install-ate.sh deploy ate-system --fake-data-plane), which +# data plane (ate-setup deploy ate-system --fake-data-plane), which # serves the WorkerPool without worker pods: fake-atelet # takes atelet's place on every atelet node and answers each # lifecycle call after a delay, and fake-workersync honors diff --git a/benchmarking/automation/util.py b/benchmarking/automation/util.py index ad6a183906..330d21b4f5 100644 --- a/benchmarking/automation/util.py +++ b/benchmarking/automation/util.py @@ -114,7 +114,7 @@ def isolate_block(test: dict) -> dict | None: def isolate_install_args(test: dict) -> list[str]: - """Return the hack/install-ate.sh flags a tests.yaml entry's isolate block + """Return the ate-setup flags a tests.yaml entry's isolate block installs the fake data plane with, or none without a block. The installer only chooses the data plane; the block's tuning goes to deploy.sh.""" return [] if isolate_block(test) is None else ["--fake-data-plane"] diff --git a/benchmarking/nighthawk-egress/README.md b/benchmarking/nighthawk-egress/README.md index f01faaf57a..3377329c26 100644 --- a/benchmarking/nighthawk-egress/README.md +++ b/benchmarking/nighthawk-egress/README.md @@ -36,19 +36,19 @@ variable when deploying ATE. 1. **Cluster with Agent Substrate deployed**: - **Existing cluster** (with `.ate-dev-env.sh` configured): ```bash - hack/install-ate.sh deploy ate-system \ + go run ./cmd/ate-setup deploy ate-system \ --credential-provider='{"name":"k8s.io"}' \ --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/nighthawk/sa/default ``` - **Local `kind` cluster**: ```bash - hack/install-ate-kind.sh deploy ate-system \ + go run ./cmd/ate-setup --kind deploy ate-system \ --credential-provider='{"enabled":false}' \ --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/nighthawk/sa/default ``` - To set the `atenet-egress` Envoy worker thread count (`--concurrency`), set `E2E_ATE_ENVOY_CONCURRENCY` when deploying: ```bash - E2E_ATE_ENVOY_CONCURRENCY=1 hack/install-ate-kind.sh deploy ate-system --credential-provider='{"name":"k8s.io"}' --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/nighthawk/sa/default + E2E_ATE_ENVOY_CONCURRENCY=1 go run ./cmd/ate-setup --kind deploy ate-system --credential-provider='{"name":"k8s.io"}' --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/nighthawk/sa/default ``` - Provision CA bundle which includes cluster's trust bundle, so that egress gateway can trust nighhawk's test server cert, run: ```bash diff --git a/benchmarking/nighthawk-ingress/README.md b/benchmarking/nighthawk-ingress/README.md index 1dd1d9bb14..21b1c75f70 100644 --- a/benchmarking/nighthawk-ingress/README.md +++ b/benchmarking/nighthawk-ingress/README.md @@ -120,7 +120,7 @@ requests 2×`envoyCpu` CPUs; the runner requests `clientConcurrency`+1). ```bash # .ate-dev-env.sh at the repo root, then: -hack/install-ate.sh deploy ate-system \ +go run ./cmd/ate-setup deploy ate-system \ --credential-provider='{"name":"k8s.io"}' \ --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner benchmarking/workloads/deploy.sh --deploy --worker-count 50 --sandbox-class gvisor @@ -149,7 +149,7 @@ Flags: `--envoy-cpu`, `--actors`, `--tail-latency-slo-ms`, `--atespace`, Notes: - The script does not redeploy substrate — if substrate code changed, - rerun `hack/install-ate.sh deploy ate-system` first. + rerun `go run ./cmd/ate-setup deploy ate-system` first. - `--atespace` isolates actor *fleets* between experiments sharing a cluster; capacity runs themselves still need the router exclusively (the cpu pin is global and the run saturates it). diff --git a/benchmarking/nighthawk-ingress/run-dev.sh b/benchmarking/nighthawk-ingress/run-dev.sh index e20dd51e1a..7990c874ca 100755 --- a/benchmarking/nighthawk-ingress/run-dev.sh +++ b/benchmarking/nighthawk-ingress/run-dev.sh @@ -80,7 +80,7 @@ docker info >/dev/null 2>&1 || { kubectl get deployment atenet-router -n ate-system >/dev/null 2>&1 || { echo "ERROR: substrate is not deployed on ${CLUSTER_NAME}. Run:" >&2 - echo " hack/install-ate.sh deploy ate-system --credential-provider='{\"name\":\"k8s.io\"}' --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner" >&2 + echo " go run ./cmd/ate-setup deploy ate-system --credential-provider='{\"name\":\"k8s.io\"}' --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/benchmarking/sa/benchmark-runner" >&2 exit 1 } diff --git a/benchmarking/telemetry/README.md b/benchmarking/telemetry/README.md index a5d991b037..e097b3e8b0 100644 --- a/benchmarking/telemetry/README.md +++ b/benchmarking/telemetry/README.md @@ -73,7 +73,7 @@ holds the same count connector and gives the same two counts. A meter in front of it would add a hop and measure nothing new. The meter exists for GKE only, where the managed collector cannot hold a connector. -`hack/install-ate.sh` installs the collector and a Prometheus with it, thus a +`ate-setup --kind` installs the collector and a Prometheus with it, thus a kind cluster needs no other step: ```bash diff --git a/cmd/ate-setup/README.md b/cmd/ate-setup/README.md index 17e1d8c248..540dc84c5f 100644 --- a/cmd/ate-setup/README.md +++ b/cmd/ate-setup/README.md @@ -7,10 +7,6 @@ go run ./cmd/ate-setup [global flags] [flags] go build -o bin/ate-setup ./cmd/ate-setup ``` -`ate-setup` is the installer. `hack/install-ate.sh` is a shim over it, kept so -that existing command lines and CI jobs keep working; it holds no install logic -of its own. - ## Installing pre-built images By default every image is built from the checkout with `ko`, which is what a @@ -37,10 +33,7 @@ the checkout, which should be the tag being installed. Each reference is pinned to the digest its tag names, so the registry has to be readable from here as well as from the cluster. -- [`commands.md`](commands.md) — every command with its `hack/install-ate.sh` - equivalent. +- [`commands.md`](commands.md) — every command, and the common global flags. - [`differences.md`](differences.md) — where the Go port deliberately behaves differently from the shell scripts it replaced, and what was reproduced exactly. -- [`cli-diff.md`](cli-diff.md) — what a `hack/install-ate.sh` user can observe - after the switch. diff --git a/cmd/ate-setup/cli-diff.md b/cmd/ate-setup/cli-diff.md deleted file mode 100644 index 5f538cd2a9..0000000000 --- a/cmd/ate-setup/cli-diff.md +++ /dev/null @@ -1,4 +0,0 @@ -# `hack/install-ate.sh` after the move to `ate-setup` - -`hack/install-ate.sh` no longer installs anything. It merely directly invokes -`go run ./cmd/ate-setup`. New logic should not be added to it. \ No newline at end of file diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index b10ce64d40..28ed5d6fcf 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -1,48 +1,43 @@ # ate-setup commands -Every `ate-setup` command alongside the equivalent `hack/install-ate.sh` flag. - -`ate-setup` is the installer. `hack/install-ate.sh` is a shim that translates -the flags below onto these commands, so an existing invocation keeps working; -this table is how to write it directly, and -[`cli-diff.md`](cli-diff.md) covers what the translation does not cover. +Every `ate-setup` command, and the common global flags. +`go run ./cmd/ate-setup --help` lists them all. ``` go run ./cmd/ate-setup [global flags] [flags] ``` -`hack/install-ate.sh` accepts its flags in any order and runs one action per -flag, in command line order. `ate-setup` runs exactly one command per -invocation, so a shell line that passed several `--deploy-*` flags becomes -several `ate-setup` calls — which is what the shim does with it. +`ate-setup` runs exactly one command per invocation. ## Global flags -These configure every command. `hack/install-ate.sh` collects its equivalents in -a pre-scan pass, so they may appear anywhere on its command line. - -| `ate-setup` | `hack/install-ate.sh` | Notes | -|---|---|---| -| `--kind` | `hack/install-ate-kind.sh`, or `ATE_INSTALL_KIND=true` | Kind overlays, the local registry, and host-architecture image builds | -| `--atenet-dataplane envoy\|agentgateway` | `--atenet-dataplane envoy\|agentgateway` | atenet ingress and egress dataplane (default `envoy`) | -| `--rollout-timeout DURATION` | `--rollout-timeout DURATION` | Readiness timeout for workloads (default `60s`). Unlike the shell flag it also governs the podcertificate-controller and CSI waits, which stay at their 120s default until it is passed | -| `--podcert-workers-per-signer N` | `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer | -| `--cluster-size size0\|size10` | `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent | -| `--ateapi-postgres-storage-class NAME` | `ATE_API_POSTGRES_STORAGE_CLASS=NAME` | StorageClass for the bundled PostgreSQL volume instead of the cluster default, such as `dynamic-rwo` on GKE so C4 and N4 nodes can attach it. An existing install must delete the postgres StatefulSet and its claim first | -| `--cordon-system-infra-components` | `--cordon-system-infra-components` | Keep the system infra components, such as ate-api-server, atenet-router, atenet-egress and postgres, off the worker nodes. Assumes a small shared pool labeled and tainted `ate.dev/workloadType=ate-system-infra:NoSchedule`, across which each workload's replicas are spread, and a one-node pool labeled and tainted `ate.dev/workloadType=ate-postgres:NoSchedule` for postgres alone. `ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS=true` when the flag is absent | -| `--fake-data-plane` | `--fake-data-plane` | Install the fake data plane for control-plane benchmarks: fake-atelet takes atelet's place in its DaemonSet, taking its objectstore-plugin's storage settings, ate-controller is scaled to zero, and fake-workersync honors every WorkerPool with fake Workers on the fake-atelet nodes. Reaches `deploy ate-system`, `deploy atelet`, and `deploy ate-controller`; only `deploy ate-system` switches a cluster between the real and fake data planes, and the other two refuse a data plane other than the installed one. `deploy ate-system --fake-data-plane` refuses a cluster whose WorkerPools have worker pods. fake-atelet runs with its defaults; a benchmark tunes it through the optional `fake-atelet-tuning` ConfigMap, whose `FAKE_ATELET_*` variables set its flags, and which no install creates or resets. Actors do not exist; never use it on a cluster that serves real actors. `ATE_INSTALL_FAKE_DATA_PLANE=true` when the flag is absent | -| `--ingress-allowed-spiffe-ids IDS` | `--ingress-allowed-spiffe-ids IDS`, or `ATE_INGRESS_ALLOWED_SPIFFE_IDS=IDS` | Comma-separated SPIFFE IDs (exact match) of the clients the ingress router lets reach actors; any listed client may reach any actor. Unset, no client may reach actors through the router. Both dataplanes require a Pod Identity CA client certificate on HTTPS and TLS CONNECT. The SPIFFE URI SAN allowlist may be checked during or after TLS, including after CONNECT acceptance, but unlisted clients must be denied before actor routing. The podidentity signer issues `spiffe://cluster.local/ns//sa/` | -| `--experimental-additional-egress-extproc-service NS/SVC:PORT` | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | -| `--credential-provider JSON` | `--credential-provider JSON`, or `ATE_CREDENTIAL_PROVIDER` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/egress-credential-injection.md`](../../docs/egress-credential-injection.md) | -| `--ateapi-log-level LEVEL` | `--ateapi-log-level LEVEL`, or `ATE_API_LOG_LEVEL=LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). The server picks up a change without restarting | -| `--otlp-endpoint URL` | `--otlp-endpoint URL`, or `ATE_OTLP_ENDPOINT=URL` | Send control plane telemetry to `URL` instead of the cluster default (see [`benchmarking/telemetry/README.md`](../../benchmarking/telemetry/README.md)) | -| `--preview GATES` | `--preview GATES`, or `ATE_PREVIEW=GATES` | Preview gates passed as `--preview` to every substrate component: comma-separated gate names, or `ALL` for all. `ATE_PREVIEW` or `config.preview` when the flag is absent. An explicit `--preview=''` turns them all off, even when `ATE_PREVIEW` is set | -| `--context NAME` | `KUBECTL_CONTEXT=NAME` | Kubeconfig context; still defaults to `KUBECTL_CONTEXT` | -| `--kubeconfig PATH` | `KUBECONFIG=PATH` | Explicit kubeconfig path | -| `--no-dev-env` | `NO_DEV_ENV=1` | Skip `.ate-dev-env.sh` at the repository root | -| `--version` / `-v` | — | New; the shell installer had no version | -| `--image-repo REPO` | — | New. Install pre-built images from `REPO` instead of building them with `ko` | -| `--image-tag TAG` | — | New. The tag those images carry. Each of the two requires the other | +These configure every command, and may appear before or after it, except +`--version`, which takes no command. +[`docs/operator-install.md`](../../docs/operator-install.md#settings) lists +every setting with its environment variable. + +| Flag | Notes | +|---|---| +| `--kind` | Kind overlays, the local registry, and host-architecture image builds. `ATE_INSTALL_KIND=true` when the flag is absent | +| `--atenet-dataplane envoy\|agentgateway` | atenet ingress and egress dataplane (default `envoy`) | +| `--rollout-timeout DURATION` | Readiness timeout for workloads (default `60s`). It also governs the podcertificate-controller and CSI waits, which stay at their 120s default until it is passed | +| `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer | +| `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent | +| `--ateapi-postgres-storage-class NAME` | StorageClass for the bundled PostgreSQL volume instead of the cluster default, such as `dynamic-rwo` on GKE so C4 and N4 nodes can attach it. An existing install must delete the postgres StatefulSet and its claim first. `ATE_API_POSTGRES_STORAGE_CLASS` when the flag is absent | +| `--cordon-system-infra-components` | Keep the system infra components, such as ate-api-server, atenet-router, atenet-egress and postgres, off the worker nodes. Assumes a small shared pool labeled and tainted `ate.dev/workloadType=ate-system-infra:NoSchedule`, across which each workload's replicas are spread, and a one-node pool labeled and tainted `ate.dev/workloadType=ate-postgres:NoSchedule` for postgres alone. `ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS=true` when the flag is absent | +| `--fake-data-plane` | Install the fake data plane for control-plane benchmarks: fake-atelet takes atelet's place in its DaemonSet, taking its objectstore-plugin's storage settings, ate-controller is scaled to zero, and fake-workersync honors every WorkerPool with fake Workers on the fake-atelet nodes. Reaches `deploy ate-system`, `deploy atelet`, and `deploy ate-controller`; only `deploy ate-system` switches a cluster between the real and fake data planes, and the other two refuse a data plane other than the installed one. `deploy ate-system --fake-data-plane` refuses a cluster whose WorkerPools have worker pods. fake-atelet runs with its defaults; a benchmark tunes it through the optional `fake-atelet-tuning` ConfigMap, whose `FAKE_ATELET_*` variables set its flags, and which no install creates or resets. Actors do not exist; never use it on a cluster that serves real actors. `ATE_INSTALL_FAKE_DATA_PLANE=true` when the flag is absent | +| `--ingress-allowed-spiffe-ids IDS` | Comma-separated SPIFFE IDs (exact match) of the clients the ingress router lets reach actors; any listed client may reach any actor. Unset, no client may reach actors through the router. Both dataplanes require a Pod Identity CA client certificate on HTTPS and TLS CONNECT. The SPIFFE URI SAN allowlist may be checked during or after TLS, including after CONNECT acceptance, but unlisted clients must be denied before actor routing. The podidentity signer issues `spiffe://cluster.local/ns//sa/`. `ATE_INGRESS_ALLOWED_SPIFFE_IDS` when the flag is absent | +| `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | +| `--credential-provider JSON` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/egress-credential-injection.md`](../../docs/egress-credential-injection.md). `ATE_CREDENTIAL_PROVIDER` when the flag is absent | +| `--ateapi-log-level LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). The server picks up a change without restarting. `ATE_API_LOG_LEVEL` when the flag is absent | +| `--otlp-endpoint URL` | Send control plane telemetry to `URL` instead of the cluster default (see [`benchmarking/telemetry/README.md`](../../benchmarking/telemetry/README.md)). `ATE_OTLP_ENDPOINT` when the flag is absent | +| `--preview GATES` | Preview gates passed as `--preview` to every substrate component: comma-separated gate names, or `ALL` for all. `ATE_PREVIEW` or `config.preview` when the flag is absent. An explicit `--preview=''` turns them all off, even when `ATE_PREVIEW` is set | +| `--context NAME` | Kubeconfig context. `KUBECTL_CONTEXT` when the flag is absent | +| `--kubeconfig PATH` | Explicit kubeconfig path. `KUBECONFIG` when the flag is absent | +| `--no-dev-env` | Skip `.ate-dev-env.sh` at the repository root. `NO_DEV_ENV=1` when the flag is absent | +| `--version` / `-v` | Print the version | +| `--image-repo REPO` | Install pre-built images from `REPO` instead of building them with `ko` | +| `--image-tag TAG` | The tag those images carry. Each of the two requires the other | Both have an environment equivalent, read when the flag is absent: `ATE_IMAGE_REPO` and `ATE_IMAGE_TAG`. @@ -56,9 +51,7 @@ replaces an existing cluster with the same name. Deploy the control plane before deploying demos or benchmarks. These commands check for `deployment/ate-controller` in the configured system namespace before creating workload resources. An existing Deployment passes this check; normal -rollout waits still diagnose workloads that cannot become ready. The shell shim -runs actions in argument order, so put `--deploy-ate-system` before demo or -benchmark deployment flags when combining them on one command line. +rollout waits still diagnose workloads that cannot become ready. ## Installing pre-built images @@ -105,17 +98,15 @@ that already names a manifest is used as written, and is not looked up. ## Deploy -| `ate-setup` | `hack/install-ate.sh` | -|---|---| -| `deploy ate-system` | `--deploy-ate-system` | -| `deploy ate-system --setup-csi=nfs` | `--deploy-ate-system --setup-csi=nfs` | -| `deploy atelet` | `--deploy-atelet` | -| `deploy apiserver` | `--deploy-ate-apiserver` | -| `deploy ate-controller` | (no shell equivalent) | -| `deploy atenet` | `--deploy-atenet` | -| `deploy podcertificate-controller` | (no shell equivalent) | -| `deploy sandboxconfig` | (no shell equivalent) | -| `deploy postgres` | (no shell equivalent) | +- `deploy ate-system` +- `deploy ate-system --setup-csi=nfs` +- `deploy atelet` +- `deploy apiserver` +- `deploy ate-controller` +- `deploy atenet` +- `deploy podcertificate-controller` +- `deploy sandboxconfig` +- `deploy postgres` `deploy ate-system` is the whole control plane: CRDs, RBAC, the store, the apiserver, the controller, atenet, and atelet. It creates every `create` @@ -124,11 +115,9 @@ a running cluster. ## Publish -| `ate-setup` | `hack/install-ate.sh` | -|---|---| -| `publish worker-images` | (no shell equivalent) | -| `publish release-images` | (no shell equivalent) | -| `publish release-binaries` | (no shell equivalent) | +- `publish worker-images` +- `publish release-images` +- `publish release-binaries` `publish worker-images` builds and pushes the worker images for the checked-out build and prints their refs; a WorkerPool points `spec.workerImage` @@ -146,11 +135,9 @@ attaches the files to the draft release. ## Delete -| `ate-setup` | `hack/install-ate.sh` | -|---|---| -| `delete ate-system` | `--delete-ate-system` | -| `delete atenet` | `--delete-atenet` | -| `delete all` | `--delete-all` | +- `delete ate-system` +- `delete atenet` +- `delete all` `delete all` removes every registered demo and then the control plane. @@ -158,73 +145,60 @@ attaches the files to the draft release. Individual secrets and config that `deploy ate-system` creates automatically. -| `ate-setup` | `hack/install-ate.sh` | -|---|---| -| `create jwt-authority-pool` | `--create-jwt-authority-pool-secret` | -| `create actor-id-ca-pool` | `--create-actor-id-ca-pool-secret` | -| `create actor-id-ca-certs` | `--create-actor-id-ca-certs-secret` | -| `create egress-mitm-ca-pool` | `--create-egress-mitm-ca-pool-secret` | -| `create podcertificate-controller-cas` | `--create-podcertificate-controller-cas` | -| `create api-server-env-vars` | `--create-api-server-env-vars` | -| `create api-authentication-config` | `--create-api-authentication-config` | -| `create api-config` | `--create-api-config` | +- `create jwt-authority-pool` +- `create actor-id-ca-pool` +- `create actor-id-ca-certs` +- `create egress-mitm-ca-pool` +- `create podcertificate-controller-cas` +- `create api-server-env-vars` +- `create api-authentication-config` +- `create api-config` ## Setup -| `ate-setup` | `hack/install-ate.sh` | -|---|---| -| `setup csi [driver]` | `--setup-csi[=DRIVER]` | +- `setup csi [driver]` -`driver` is one of `nfs`, `hostpath`, `both`, or `none`; `setup csi` with `none` as the default option. The hostpath is for KIND clusters only. NFS has no such +`driver` is one of `nfs`, `hostpath`, `both`, or `none`, and defaults to `nfs`. The hostpath is for KIND clusters only. NFS has no such restriction, but it does need the `nfsd` kernel module loaded on the nodes. ## Benchmarks -| `ate-setup` | `hack/install-ate.sh` | -|---|---| -| `deploy benchmarks` | `--deploy-benchmarks` | -| `delete benchmarks` | `--delete-benchmarks` | -| `--worker-count N` | `--benchmark-worker-count N` (default `1`) | -| `--sandbox-class gvisor\|microvm` | `--benchmark-sandbox-class CLASS` (default `gvisor`) | -| `BENCHMARK_ACTOR_MEMORY=SIZE` | `--benchmark-actor-memory SIZE` (default `256Mi`) | - -The memory limit has no flag: `benchmarking/workloads/deploy.sh` has always -taken it from the environment, and the shim exports it. - -The other two flags are per-command in `ate-setup` and global in -`hack/install-ate.sh`, which forwards them to whichever benchmark action runs. -See +- `deploy benchmarks` +- `delete benchmarks` +- `--worker-count N` (default `1`) +- `--sandbox-class gvisor|microvm` (default `gvisor`) +- `--benchmark-actor-memory SIZE` (default `256Mi`) + +`--benchmark-actor-memory` is a global flag (`BENCHMARK_ACTOR_MEMORY` when it +is absent); the other two are flags of the benchmark commands. See [`benchmarking/README.md`](../../benchmarking/README.md). ## Demos -| `ate-setup` | `hack/install-ate.sh` | +- `deploy demo NAME` +- `delete demo NAME` + +`NAME` is one of: + +| Command | Description | |---|---| -| `deploy demo NAME` | `--deploy-demo-NAME` | -| `delete demo NAME` | `--delete-demo-NAME` | - -`NAME` is the demo without its `demo-` prefix: - -| `ate-setup` | `hack/install-ate.sh` | Description | -|---|---|---| -| `deploy demo counter` | `--deploy-demo-counter` | A counter actor exercising snapshot, resume, and atenet ingress | -| `deploy demo counter --with-external-volume [--storage-class NAME]` | `--deploy-demo-counter-with-external-volume` (`STORAGE_CLASS=NAME`) | The same, plus an external volume and a pre-seeded file to validate (requires `--preview=ExternalVolumes`). Run `setup csi` first and name the class it created, e.g. `csi-nfs-sc`; defaults to `standard` | -| `deploy demo counter-microvm` | `--deploy-demo-counter-microvm` | The counter demo on microVM workers. Run `hack/install-microvm-deps.sh --install` first | -| `deploy demo egress` | `--deploy-demo-egress` | Egress policy enforcement through atenet; the actors trust the egress gateway's CA | -| `deploy demo egress-microvm` | `--deploy-demo-egress-microvm` | The same on microVM workers. Run `hack/install-microvm-deps.sh --install` first | -| `deploy demo jupyter` | `--deploy-demo-jupyter` | A Jupyter notebook server per actor, reached through atenet ingress | -| `deploy demo sandbox` | `--deploy-demo-sandbox` | An on-demand sandbox actor driven by the sandbox client | -| `deploy demo multi-template` | `--deploy-demo-multi-template` | Two ActorTemplates sharing one WorkerPool | -| `deploy demo parking` | `--deploy-demo-parking` | Actor parking and unparking on a small WorkerPool | -| `deploy demo claude-code-multiplex` | `--deploy-demo-claude-code-multiplex` | Several Claude Code agents multiplexed onto one WorkerPool (requires `ANTHROPIC_API_KEY`, `BUCKET_NAME`, `KO_DOCKER_REPO`) | - -Each has a matching `delete demo NAME` / `--delete-demo-NAME`. Demo flags bind +| `deploy demo counter` | A counter actor exercising snapshot, resume, and atenet ingress | +| `deploy demo counter --with-external-volume [--storage-class NAME]` | The same, plus an external volume and a pre-seeded file to validate (requires `--preview=ExternalVolumes`). Run `setup csi` first and name the class it created, e.g. `csi-nfs-sc`; defaults to `standard` | +| `deploy demo counter-microvm` | The counter demo on microVM workers. Run `hack/install-microvm-deps.sh --install` first | +| `deploy demo egress` | Egress policy enforcement through atenet; the actors trust the egress gateway's CA | +| `deploy demo egress-microvm` | The same on microVM workers. Run `hack/install-microvm-deps.sh --install` first | +| `deploy demo jupyter` | A Jupyter notebook server per actor, reached through atenet ingress | +| `deploy demo sandbox` | An on-demand sandbox actor driven by the sandbox client | +| `deploy demo multi-template` | Two ActorTemplates sharing one WorkerPool | +| `deploy demo parking` | Actor parking and unparking on a small WorkerPool | +| `deploy demo claude-code-multiplex` | Several Claude Code agents multiplexed onto one WorkerPool (requires `ANTHROPIC_API_KEY`, `BUCKET_NAME`, `KO_DOCKER_REPO`) | + +Each has a matching `delete demo NAME`. Demo flags bind to the deploy side only; teardown never reads them. The demo list is not hard-coded here — it is built from the registry in [`internal/demos`](internal/demos), one package per demo, so `go run ./cmd/ate-setup deploy demo --help` is authoritative for both the list and the per-demo flags. A new demo is a new package there, added to -[`internal/demos/all`](internal/demos/all) and mirrored into the `ATE_DEMOS` -list in `hack/install-ate.sh`, which a test keeps in step with the registry. +[`internal/demos/all`](internal/demos/all). diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 12209ac94b..d97a6bbdb7 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -3,17 +3,13 @@ `ate-setup` is a Go port of `hack/install-ate.sh`, the seven `hack/install-demo-*.sh` scripts it sourced, and `hack/setup-csi-*-kind.sh`. -The port has replaced them. The demo scripts are deleted, and -`hack/install-ate.sh` is a shim holding no install logic of its own: it -translates the flags and environment variables the installer has always -accepted onto `ate-setup` commands, so existing command lines keep working. See -[`commands.md`](commands.md) for the flag-by-flag mapping. - -This document covers what changed *behind* that mapping: the design decisions -of the port, against the shell it replaced. For what a user of -`hack/install-ate.sh` can observe after the switch, see -[`cli-diff.md`](cli-diff.md). For anything listed in neither, the port is -intended to be behavior-preserving. +The port has replaced them. `hack/install-ate.sh`, the demo scripts, and +`hack/setup-csi-hostpath-kind.sh` are deleted. See +[`commands.md`](commands.md) for the commands. + +This document covers the design decisions of the port, against the shell it +replaced. For anything not listed here, the port is intended to be +behavior-preserving. ## What is deliberately unchanged @@ -21,8 +17,7 @@ These were treated as contracts and reproduced exactly: - **Step log lines.** `log.Step` prints the same cyan `[step]: name` and the same step names (`deploy_ate_system`, `create_api_server_env_vars`, - `demo-counter_deploy`), so CI log scrapers keep working. Two lines lost a - parenthesized qualifier; see [`cli-diff.md`](cli-diff.md). + `demo-counter_deploy`), so CI log scrapers keep working. - **Manifest ordering.** Applying a directory is non-recursive and lexical, as `kubectl apply -f ` was. `delete_ate_system` depends on that ordering and the shell comments called out specific filename hazards, so @@ -58,8 +53,7 @@ These were treated as contracts and reproduced exactly: | Repository root | `git rev-parse --show-toplevel`, then `cd` | walk up for `go.mod`; no `chdir`, all paths absolute | The one-action-per-run change is the most visible: a line that passed -`--deploy-ate-system --deploy-demo-counter` becomes two `ate-setup` calls. That -is what the shim does with it, so the combined form keeps working. +`--deploy-ate-system --deploy-demo-counter` becomes two `ate-setup` calls. Invalid input now fails before any cluster mutation. `--atenet-dataplane=nginx` used to be caught by a pre-scan validation pass; `--worker-count 0` was not @@ -186,7 +180,7 @@ teardown after the CRDs are gone must not fail. Beyond that, deletes are strict. **Cluster profiles are new.** `--cluster-size` and `--cordon-system-infra-components` -postdate the shell installer, which the shim forwards them to. `size10` +postdate the shell installer. `size10` renders the `podcert-size10` overlay and resizes the bundled PostgreSQL from `postgres-size10/postgres-config-patch.yaml`; `--cordon-system-infra-components` composes the diff --git a/cmd/ate-setup/internal/cmd/csi_test.go b/cmd/ate-setup/internal/cmd/csi_test.go index 393a4dfc5d..106d67ece8 100644 --- a/cmd/ate-setup/internal/cmd/csi_test.go +++ b/cmd/ate-setup/internal/cmd/csi_test.go @@ -81,7 +81,7 @@ func TestAnUnknownCSIDriverIsRejectedBeforeAnythingRuns(t *testing.T) { // The accepted spellings have to make it past the same check, including the // bare `setup csi` that takes the default. func TestAcceptedCSIDriversPassArgumentValidation(t *testing.T) { - accepted := []string{"nfs", "hostpath", "both", "none", "true", "false", ""} + accepted := []string{"nfs", "hostpath", "both", "none"} t.Run("setup csi", func(t *testing.T) { for _, driver := range accepted { diff --git a/cmd/ate-setup/internal/cmd/root.go b/cmd/ate-setup/internal/cmd/root.go index 4fa51941ed..a33a97cf7b 100644 --- a/cmd/ate-setup/internal/cmd/root.go +++ b/cmd/ate-setup/internal/cmd/root.go @@ -216,10 +216,7 @@ func recordRun(cmd *cobra.Command, runErr error) { } } -// Root is the command tree, for callers that need to inspect it rather than -// run it. hack/install-ate.sh translates the historical installer flags into -// ate-setup command lines, and its test resolves each one against this to check -// it still names a command that exists. +// Root is the command tree, for tests that run it in-process. func Root() *cobra.Command { return rootCmd } // Execute runs the command tree. @@ -234,6 +231,11 @@ func Execute() { // ExecuteC reports which command ran, which the failure record needs and // Execute does not return. executed, err := rootCmd.ExecuteC() + // Cobra rejects an unknown command only at the root. Below it, a command + // that only groups others prints its help for any argument and succeeds. + if err == nil && !executed.Runnable() && executed.Flags().NArg() > 0 { + err = fmt.Errorf("unknown command %q for %q", executed.Flags().Arg(0), executed.CommandPath()) + } if err != nil { fmt.Fprintln(os.Stderr, "Error:", err) // After the error, so the retry line is the last thing on screen: diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 7a80966d89..bf300d7469 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -429,10 +429,9 @@ func (c *Config) Path(parts ...string) string { return filepath.Join(append([]string{c.Root}, parts...)...) } -// kindUnsetVars are the GKE-specific variables the shell kind installer unset -// before delegating, so that a developer who had already sourced -// .ate-dev-env.sh into their shell did not end up pointing a local install at a -// cloud project. +// kindUnsetVars are the GKE-specific variables a Kind install drops, so that a +// developer who has already sourced .ate-dev-env.sh into their shell does not +// point a local install at a cloud project. var kindUnsetVars = []string{ "GCE_REGION", "CLUSTER_LOCATION", "NETWORK", "SUBNETWORK", "MEMORYSTORE_INSTANCE", "PROJECT_ID", diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index b50366e1d9..074fa5f99e 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -859,9 +859,8 @@ func TestLoadOtlpEndpoint(t *testing.T) { } } -// hack/install-ate-kind.sh exports ATE_INSTALL_KIND rather than passing a flag, -// so the environment has to select the Kind profile as completely as --kind -// does; a Kind install that only half-applied would push images to the wrong +// ATE_INSTALL_KIND has to select the Kind profile as completely as --kind does; +// a Kind install that only half-applied would push images to the wrong // registry. func TestLoadKindFromEnvironment(t *testing.T) { loadEnv(t) @@ -1178,8 +1177,8 @@ func TestLoadEnvoyConcurrency(t *testing.T) { } } -// --kind must reproduce the exports in the shell kind installer, including -// clearing the GKE coordinates and isolating the snapshot bucket name. +// --kind must apply the whole Kind profile, including clearing the GKE +// coordinates and isolating the snapshot bucket name. func TestLoadKindProfile(t *testing.T) { loadEnv(t) t.Setenv("PROJECT_ID", "some-gke-project") diff --git a/cmd/ate-setup/internal/config/file.go b/cmd/ate-setup/internal/config/file.go index 542628e7a0..683f94a8a5 100644 --- a/cmd/ate-setup/internal/config/file.go +++ b/cmd/ate-setup/internal/config/file.go @@ -61,8 +61,7 @@ func ParseFile(path string) (*File, error) { raw, err := os.ReadFile(path) if err != nil { // Name the absolute path: a relative one means different files from - // different working directories, and the shell installer changes to - // the repository root before running. + // different working directories. abs, absErr := filepath.Abs(path) if absErr != nil { abs = path diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index fa16bd5176..c553243a67 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -31,10 +31,6 @@ import ( ) // ConfigPathEnv names the configuration document when --config is not given. -// -// It exists so the file channel crosses hack/install-ate.sh, which forwards -// only a fixed set of flags but passes the whole environment to the child -// process. Every other setting reaches ate-setup that way already. const ConfigPathEnv = "ATE_CONFIG" // LoadOptions carries the inputs that select where configuration comes from, @@ -85,11 +81,9 @@ func LoadFlags(fs *pflag.FlagSet, o LoadOptions) (*Config, error) { // configPathFrom picks the configuration document, flag over environment. // -// A relative path resolves against the working directory, which is the -// repository root when the shell installer invoked this: it does cd "$ROOT" -// before running the binary. ParseFile reports the absolute path it tried, so -// a path that meant something else from the caller's directory is visible in -// the error rather than silently missing. +// A relative path resolves against the working directory. ParseFile reports +// the absolute path it tried, so a path that meant something else from the +// caller's directory is visible in the error rather than silently missing. func configPathFrom(o LoadOptions) string { if o.ConfigPath != "" { return o.ConfigPath @@ -143,9 +137,8 @@ func resolveKind(fs *pflag.FlagSet, file *File, env map[string]string) bool { // mergeDevEnv layers .ate-dev-env.sh underneath the process environment. // -// Sourcing is skipped for Kind installs the same way the shell kind installer -// exports NO_DEV_ENV: the GKE-shaped variables in a developer's file would -// otherwise point a local install at a cloud project. +// Sourcing is skipped for Kind installs: the GKE-shaped variables in a +// developer's file would otherwise point a local install at a cloud project. func mergeDevEnv(root string, env map[string]string, noDevEnv, kind bool) error { if noDevEnv || kind || os.Getenv("NO_DEV_ENV") != "" { return nil diff --git a/cmd/ate-setup/internal/config/load_test.go b/cmd/ate-setup/internal/config/load_test.go index b48f8b3ebe..0ac2c83c29 100644 --- a/cmd/ate-setup/internal/config/load_test.go +++ b/cmd/ate-setup/internal/config/load_test.go @@ -128,9 +128,8 @@ func TestLoadFlagsRejectsBadConfigPath(t *testing.T) { } } -// The file channel must cross hack/install-ate.sh, which forwards only a fixed -// set of flags but passes the whole environment through. ATE_CONFIG is how it -// gets there without a shim change. +// ATE_CONFIG names the document when --config is absent, and the flag +// outranks it. func TestConfigPathFromEnvironment(t *testing.T) { loadEnv(t) path := filepath.Join(t.TempDir(), "install.yaml") @@ -191,9 +190,8 @@ func TestConfigPathFromEnvironment(t *testing.T) { }) } -// A relative path means different files from different working directories, -// and the shell installer changes to the repository root. The error names the -// absolute path so that is visible rather than silent. +// A relative path means different files from different working directories. +// The error names the absolute path so that is visible rather than silent. func TestMissingConfigErrorNamesAbsolutePath(t *testing.T) { loadEnv(t) t.Setenv(ConfigPathEnv, "no-such-config.yaml") diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 481f72de1e..1fc1e4e3d0 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -312,7 +312,7 @@ func (e *Env) EnsureEnvVarsSafeStandalone(ctx context.Context) error { } return fmt.Errorf("the running ate-api-server Deployment does not reference the %s Secret; "+ "rewriting the env vars alone would leave it without a DSN on its next restart. "+ - "Run `ate-setup deploy ate-apiserver` instead, which also updates the Deployment", SecretAPIEnvVars) + "Run `ate-setup deploy apiserver` instead, which also updates the Deployment", SecretAPIEnvVars) } // annotateAPIServerEnvHash stamps the running Deployment's pod template with diff --git a/cmd/ate-setup/internal/steps/csi.go b/cmd/ate-setup/internal/steps/csi.go index f42183257b..f93b8bc779 100644 --- a/cmd/ate-setup/internal/steps/csi.go +++ b/cmd/ate-setup/internal/steps/csi.go @@ -69,17 +69,12 @@ type csiDriverSpec struct { // installsNothing reports whether the value selects no driver at all. func (s csiDriverSpec) installsNothing() bool { return !s.hostpath && !s.nfs } -// csiDrivers is every accepted --setup-csi value. "true", "false" and the empty -// string are the shell installer's boolean spellings, from before the flag grew -// driver names, and still work. +// csiDrivers is every accepted --setup-csi value. var csiDrivers = map[string]csiDriverSpec{ "nfs": {nfs: true}, "hostpath": {hostpath: true}, "both": {hostpath: true, nfs: true}, - "true": {hostpath: true, nfs: true}, "none": {}, - "false": {}, - "": {}, } // ValidateCSIDriver reports whether driver names something --setup-csi accepts. diff --git a/cmd/ate-setup/internal/steps/csi_test.go b/cmd/ate-setup/internal/steps/csi_test.go index 695593f140..21b56e4c9b 100644 --- a/cmd/ate-setup/internal/steps/csi_test.go +++ b/cmd/ate-setup/internal/steps/csi_test.go @@ -59,12 +59,10 @@ func TestSetupCSIDecidesBeforeTouchingTheCluster(t *testing.T) { error string }{ {name: "none", driver: "none"}, - {name: "the shell installer's false", driver: "false"}, - {name: "unset", driver: ""}, {name: "hostpath off Kind", driver: "hostpath", error: "only supported for Kind"}, {name: "both off Kind", driver: "both", error: "only supported for Kind"}, - {name: "the shell installer's true off Kind", driver: "true", error: "only supported for Kind"}, {name: "unknown driver", driver: "nfsv4", error: `unknown CSI driver "nfsv4"`}, + {name: "the shell installer's true", driver: "true", error: `unknown CSI driver "true"`}, } { t.Run(tc.name, func(t *testing.T) { e := &Env{Cfg: &config.Config{Kind: tc.kind}} @@ -170,9 +168,7 @@ func TestTheNFSSetupsApplyTheSharedStorageClass(t *testing.T) { } } -// Both spellings of each driver selection have to keep selecting the same -// thing: --setup-csi was a boolean in the shell installer, and the scripts and -// dev-env files that still pass true or false have to keep working. +// Each --setup-csi value selects the drivers its name says. func TestCSIDriverSpellings(t *testing.T) { for _, tc := range []struct { driver string @@ -181,10 +177,7 @@ func TestCSIDriverSpellings(t *testing.T) { {"nfs", csiDriverSpec{nfs: true}}, {"hostpath", csiDriverSpec{hostpath: true}}, {"both", csiDriverSpec{hostpath: true, nfs: true}}, - {"true", csiDriverSpec{hostpath: true, nfs: true}}, {"none", csiDriverSpec{}}, - {"false", csiDriverSpec{}}, - {"", csiDriverSpec{}}, } { t.Run(tc.driver, func(t *testing.T) { got, ok := csiDrivers[tc.driver] diff --git a/cmd/ate-setup/main.go b/cmd/ate-setup/main.go index d115f75c5c..2f8c227b3d 100644 --- a/cmd/ate-setup/main.go +++ b/cmd/ate-setup/main.go @@ -13,10 +13,7 @@ // limitations under the License. // Command ate-setup installs and tears down Agent Substrate on a Kubernetes -// cluster. It is the installer; hack/install-ate.sh is a shim that translates -// its historical flags onto these commands. See cmd/ate-setup/commands.md for -// the flag-by-flag mapping and cmd/ate-setup/cli-diff.md for what the -// translation does not cover. +// cluster. See cmd/ate-setup/commands.md for its commands. package main import ( diff --git a/cmd/ate-setup/preflight_test.go b/cmd/ate-setup/preflight_test.go index 729907dc9e..8ac37e7f4e 100644 --- a/cmd/ate-setup/preflight_test.go +++ b/cmd/ate-setup/preflight_test.go @@ -40,9 +40,8 @@ func repoRoot(t *testing.T) string { return strings.TrimSpace(string(out)) } -// Each invocation gets its own command tree and flags, including invocations -// made by the real shell shims. The test executable runs the installer without -// rebuilding it or contacting a real cluster. +// Each invocation gets its own command tree and flags. The test executable runs +// the installer without rebuilding it or contacting a real cluster. func TestInstallerPreflightProcess(t *testing.T) { if os.Getenv("ATE_SETUP_PREFLIGHT_TEST") != "1" { return @@ -66,17 +65,6 @@ func runPreflightInstaller(t *testing.T, kubeconfig, dockerStatus string, args . } for name, source := range map[string]string{ "docker": "#!/bin/sh\necho called >> \"$PREFLIGHT_DOCKER_LOG\"\nexit \"$PREFLIGHT_DOCKER_STATUS\"\n", - "go": `#!/bin/sh -if [ "$*" = "env GOARCH" ]; then - echo amd64 -elif [ "$1" = run ] && [ "$2" = ./cmd/ate-setup ]; then - shift 2 - exec "$PREFLIGHT_EXECUTABLE" -test.run='^TestInstallerPreflightProcess$' -- "$@" -else - echo "unexpected go invocation: $*" >&2 - exit 1 -fi -`, } { if err := os.WriteFile(filepath.Join(bin, name), []byte(source), 0o755); err != nil { t.Fatal(err) @@ -93,7 +81,6 @@ fi "NO_DEV_ENV=true", "ATE_NO_REPORT_DEFAULTS=1", "ATE_SETUP_PREFLIGHT_TEST=1", - "PREFLIGHT_EXECUTABLE=" + executable, "PREFLIGHT_DOCKER_STATUS=" + dockerStatus, "PREFLIGHT_DOCKER_LOG=" + filepath.Join(bin, "docker.log"), } @@ -197,6 +184,21 @@ func TestKindInstallerHelpNeedsNoCluster(t *testing.T) { } } +// Cobra rejects an unknown command only at the root, so a typo one level down +// used to print help and exit 0 having installed nothing. +func TestInstallerRejectsUnknownSubcommand(t *testing.T) { + for _, args := range [][]string{ + {"deploy", "demo-counter"}, + {"deploy", "demo", "demo-counter"}, + } { + out, code := runPreflightInstaller(t, preflightKubeconfig(t, ""), "0", args...) + want := `unknown command "demo-counter"` + if code == 0 || !strings.Contains(out, want) { + t.Errorf("%v: exit=%d, want an error containing %q\n%s", args, code, want, out) + } + } +} + func TestInstallerControlPlaneCheckPreservesErrorsAndAllowsInstalledSystem(t *testing.T) { for _, installed := range []bool{false, true} { t.Run(fmt.Sprintf("installed=%t", installed), func(t *testing.T) { diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index 340d696e26..2c86821b69 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -49,7 +49,7 @@ gcloud beta container clusters update CLUSTER_NAME \ **Local (kind):** -The kind overlay installed by `hack/install-ate-kind.sh --deploy-ate-system` already provisions an in-cluster OpenTelemetry Collector and Jaeger in the `otel-system` namespace. No additional setup is required. +The kind overlay installed by `ate-setup --kind deploy ate-system` already provisions an in-cluster OpenTelemetry Collector and Jaeger in the `otel-system` namespace. No additional setup is required. Port-forward the Jaeger UI and invoke any command with `--trace`: ```bash diff --git a/cmd/kubectl-ate/internal/cmd/actortemplate_test.go b/cmd/kubectl-ate/internal/cmd/actortemplate_test.go index 5ce5d737fa..6ca43a6d9f 100644 --- a/cmd/kubectl-ate/internal/cmd/actortemplate_test.go +++ b/cmd/kubectl-ate/internal/cmd/actortemplate_test.go @@ -158,7 +158,7 @@ func TestActorTemplateFromManifest_DemoManifests(t *testing.T) { if err != nil { t.Fatalf("reading demo manifest: %v", err) } - // The install scripts substitute the bucket and drop the unused + // ate-setup substitutes the bucket and drops the unused // optional placeholder lines before applying. rendered := strings.ReplaceAll(string(data), "${BUCKET_NAME}", "ate-snapshots") var kept []string diff --git a/demos/claude-code-multiplex/README.md b/demos/claude-code-multiplex/README.md index 1e2c4c930d..6c9598b5ad 100644 --- a/demos/claude-code-multiplex/README.md +++ b/demos/claude-code-multiplex/README.md @@ -26,7 +26,8 @@ This guide assumes you know Kubernetes and the general shape of agent runtimes ( ## Prerequisites -- A Kubernetes cluster with **Agent Substrate** installed. +- A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) + (with `ATE_INSTALL_KIND=true` exported in your shell) or [GKE Quickstart](../../README.md#gke-quickstart-development). - `kubectl` configured against that cluster (the dashboard uses the operator's kubeconfig via [`client-go`](https://github.com/kubernetes/client-go) for pod-log reads). - Network reach to the substrate **api** gRPC service (`api.ate-system:443`). If you run the dashboard as a pod in the cluster, label its namespace `ate.dev/api-access=true`, because ate-system's NetworkPolicies admit API clients only from labeled namespaces (see [`ate-system-networkpolicies.yaml`](../../manifests/ate-install/ate-system-networkpolicies.yaml)). When running the dashboard from outside the cluster, port-forward it in a separate terminal (same convention as [`demos/sandbox/README.md`](../sandbox/README.md#3-port-forward-services)) and keep it running for the lifetime of the demo: ```bash @@ -34,8 +35,7 @@ This guide assumes you know Kubernetes and the general shape of agent runtimes ( kubectl port-forward svc/api 8080:443 -n ate-system ``` - An **Anthropic API key** (the agents call Claude). -- A GCS bucket for substrate state snapshots (configured during Substrate install). -- `KO_DOCKER_REPO` set to a registry you can push to (e.g. `gcr.io/${PROJECT_ID}/ate-images`, same as `hack/ate-dev-env.sh.example`). The deploy step builds and pushes the workload image there with a sha256-pinned reference. +- The snapshot bucket and image registry (`BUCKET_NAME`, `KO_DOCKER_REPO`) that your Substrate install uses. The deploy step builds and pushes the workload image to that registry with a sha256-pinned reference. - `docker buildx` (the deploy function builds the workload image — a Dockerfile-based Python + Claude Code wrapper, not a Go binary, so `ko` doesn't apply for the workload itself). ## Components @@ -44,7 +44,7 @@ This guide assumes you know Kubernetes and the general shape of agent runtimes ( |---|---| | `demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl` | Namespace and WorkerPool manifest | | `demos/claude-code-multiplex/agent-*-template.yaml.tmpl` | One protojson ActorTemplate per agent, created through the ate API with `kubectl ate create actor-template` | -| `cmd/ate-setup/internal/demos/claudemultiplex/` | The installer's demo package, behind `--deploy-demo-claude-code-multiplex` / `--delete-demo-claude-code-multiplex` | +| `cmd/ate-setup/internal/demos/claudemultiplex/` | The installer's demo package, behind `deploy demo claude-code-multiplex` / `delete demo claude-code-multiplex` | | `demos/claude-code-multiplex/workload/` | The agent container image source (Dockerfile + entrypoint that wires Claude Code; built and pushed by the deploy step) | | `demos/claude-code-multiplex/ui/` | Static dashboard (`index.html` + `server.go`) that talks to the cluster | @@ -52,11 +52,10 @@ This guide assumes you know Kubernetes and the general shape of agent runtimes ( ### 1. Deploy the demo -From the repo root, with your Anthropic key and substrate bucket name in the environment: +From the repo root, with your Anthropic key in the environment: ```bash ANTHROPIC_API_KEY=sk-ant-... \ -BUCKET_NAME=your-substrate-bucket \ go run ./cmd/ate-setup deploy demo claude-code-multiplex ``` diff --git a/demos/counter/README.md b/demos/counter/README.md index 4886c7d222..604b75dbe5 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -9,17 +9,19 @@ The demo uses two kinds of resources: the `WorkerPool` is a Kubernetes CRD, whil ## Prerequisites - A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) - or [GKE Quickstart](../../README.md#gke-quickstart-development). -- `ko` installed for building images. -- A GCS bucket for storing snapshots (configured via `BUCKET_NAME` env var). + (with `ATE_INSTALL_KIND=true` exported in your shell) or [GKE Quickstart](../../README.md#gke-quickstart-development). ## How to Run on Agent Substrate +The [Quickstart](../../README.md#quickstart-development) already deploys this +demo, creates `my-counter-1` and starts its proxy on port `8000`. If you ran +it, skip to [How to Use](#how-to-use). + ### 1. Build and Deploy > [!NOTE] > Do not manually edit `demos/counter/counter.yaml.tmpl` or -> `demos/counter/counter-template.yaml.tmpl`. The installation script +> `demos/counter/counter-template.yaml.tmpl`. `ate-setup` > automatically injects your `${BUCKET_NAME}` environment variable during > deployment. @@ -59,17 +61,17 @@ go run ./cmd/ate-setup deploy demo counter-microvm
Installation on GVisor -Use the core installation script to build the image and deploy the demo to your cluster: +Use `ate-setup` to build the image and deploy the demo to your cluster: ```bash -./hack/install-ate.sh --deploy-demo-counter +go run ./cmd/ate-setup deploy demo counter ``` To enable validation of reading from an external volume (e.g. `/external-data/test.txt`), run: ```bash -./hack/install-ate.sh --deploy-demo-counter-with-external-volume --preview=ExternalVolumes +go run ./cmd/ate-setup deploy demo counter --with-external-volume --preview=ExternalVolumes ``` This command will: @@ -209,7 +211,8 @@ kubectl ate delete actor my-counter-1 -a ate-demo-counter `counter.go` also listens on a second, configurable port (`--extra-port=9090` in this demo's template) to demonstrate atenet-router's arbitrary-port ingress support. Pass `[local-port:]remote-port` arguments to `kubectl ate proxy actor` -to forward additional ports: +to forward additional ports. This proxy also takes port `8000`, so stop the +earlier one (such as the Quickstart's) first:
Non standard port with MicroVM diff --git a/demos/counter/counter-template.yaml.tmpl b/demos/counter/counter-template.yaml.tmpl index 9f53b504c5..b92c17ee2a 100644 --- a/demos/counter/counter-template.yaml.tmpl +++ b/demos/counter/counter-template.yaml.tmpl @@ -18,7 +18,7 @@ # atespace must exist. # # The ${...}_ARG/MOUNTS/VOLUMES placeholder lines are dropped on a plain -# deploy and filled in by --deploy-demo-counter-with-external-volume. +# deploy and filled in by `deploy demo counter --with-external-volume`. metadata: atespace: ate-demo-counter name: counter diff --git a/demos/egress/README.md b/demos/egress/README.md index 923c0c2647..98cb7809c5 100644 --- a/demos/egress/README.md +++ b/demos/egress/README.md @@ -53,18 +53,11 @@ intercepted and carried over mTLS to a gateway that verifies who is making the r ## Choose a dataplane -The dataplane is selected when installing `ate-system`, not when installing this demo. The Actor, -ActorTemplate, worker pool, test, and manual walkthrough are otherwise the same. - -```bash -# Envoy (default) -./hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' \ - --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-system/sa/atenet-client - -# agentgateway -./hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=agentgateway --credential-provider='{"name":"k8s.io"}' \ - --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-system/sa/atenet-client -``` +The dataplane is selected when installing `ate-system`, not when installing this demo. Install as +in the [Quickstart](../../README.md#quickstart-development) or +[GKE Quickstart](../../README.md#gke-quickstart-development); for agentgateway, add +`--atenet-dataplane=agentgateway` to `deploy ate-system`. The Actor, ActorTemplate, worker pool, +test, and manual walkthrough are otherwise the same. | | Envoy | agentgateway | | --- | --- | --- | @@ -94,15 +87,15 @@ rejects that option with agentgateway rather than silently omitting it. (step 3 below) and `kubectl ate get egress-policy` reads it back; the e2e suites create theirs with `e2e.EnsureEgressPolicy`. An `all` rule reproduces the pre-policy behavior. - **Actor-identity trust** — the gateway mounts the `actor-id-ca-certs` Secret, a cert-only copy of - the actor-identity CA root that `hack/install-ate.sh` derives from `actor-id-ca-pool` (which also + the actor-identity CA root that `ate-setup` derives from `actor-id-ca-pool` (which also holds the CA signing key and is deliberately *not* mounted here). ## Prerequisites -- A kind cluster with Agent Substrate installed (`hack/create-kind-cluster.sh`, followed by one of - the [dataplane installation commands above](#choose-a-dataplane)). Egress is enabled by the - ateapi flag above. -- `ko`, `kubectl`, and `kubectl-ate` (`go install ./cmd/kubectl-ate`). +- A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) + (with `ATE_INSTALL_KIND=true` exported in your shell) or [GKE Quickstart](../../README.md#gke-quickstart-development), with the dataplane + [chosen above](#choose-a-dataplane). Egress is enabled by the ateapi flag above. +- `kubectl` and `kubectl-ate` (`go install ./cmd/kubectl-ate`). ## Deploy the demo fixture diff --git a/demos/egress/test-egress.sh b/demos/egress/test-egress.sh index 83a4cc3ae4..d88e6169cd 100755 --- a/demos/egress/test-egress.sh +++ b/demos/egress/test-egress.sh @@ -23,7 +23,7 @@ # certificate, cannot open a tunnel: the gateway trusts the actor-identity # CA, so the mTLS handshake itself is refused. # -# Prerequisites: a substrate cluster with `--deploy-demo-egress` applied, plus +# Prerequisites: a substrate cluster with `ate-setup deploy demo egress` applied, plus # kubectl and kubectl-ate on PATH. The install's --ingress-allowed-spiffe-ids # must include spiffe://cluster.local/ns/ate-system/sa/atenet-client. See demos/egress/README.md. # @@ -33,7 +33,12 @@ set -o errexit -o nounset -o pipefail -CTX="${KUBECTL_CONTEXT:-kind-kind}" +# Target the cluster ate-setup deploys to: KUBECTL_CONTEXT, else the kind +# cluster when ATE_INSTALL_KIND=true, else the current context. +if [[ -z "${KUBECTL_CONTEXT:-}" && "${ATE_INSTALL_KIND:-}" == "true" ]]; then + KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME:-kind}" +fi +CTX="${KUBECTL_CONTEXT:-$(kubectl config current-context)}" # The actor lives in the demo's atespace: --template resolves the # template by name within the actor's own atespace. ATESPACE="${ATESPACE:-ate-demo-egress}" diff --git a/demos/jupyter/README.md b/demos/jupyter/README.md index 8931321ab9..2fdd7f5faf 100644 --- a/demos/jupyter/README.md +++ b/demos/jupyter/README.md @@ -4,33 +4,14 @@ This demo runs a Jupyter Notebook on Agent Substrate. It demonstrates how standa ## Prerequisites -- A local Kubernetes cluster via Kind (or GKE). -- `ko` installed for building images. -- A GCS bucket for storing snapshots, or an in-cluster snapshot repository (handled automatically by the kind install script). +- A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) + (with `ATE_INSTALL_KIND=true` exported in your shell) or [GKE Quickstart](../../README.md#gke-quickstart-development). -## Quickstart on Kind +## Deploy the Jupyter Notebook Demo -For local development, Agent Substrate provides helper scripts to get a Kind cluster running and configured. - -1. **Create the Kind cluster:** - - ```bash - ./hack/create-kind-cluster.sh - # This will create a local cluster named "ate-demo" and configure a local registry. - ``` - -2. **Install Agent Substrate core system to the Kind cluster:** - - ```bash - ./hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' \ - --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-system/sa/atenet-client - ``` - -3. **Deploy the Jupyter Notebook Demo:** - - ```bash - ./hack/install-ate-kind.sh --deploy-demo-jupyter - ``` +```bash +go run ./cmd/ate-setup deploy demo jupyter +``` ## Creating and Accessing Your Jupyter Actor @@ -99,10 +80,5 @@ Or, even easier, you can rely on "transparent resume" — just refresh the page Uninstall the demo deployment — this deletes the demo's actors, the actor template, and the `ate-demo-jupyter` atespace: ```bash -./hack/install-ate-kind.sh --delete-demo-jupyter -``` - -Uninstall the entire local Kind cluster: -```bash -./hack/delete-kind-cluster.sh +go run ./cmd/ate-setup delete demo jupyter ``` diff --git a/demos/multi-template/README.md b/demos/multi-template/README.md index 0ac1e739e9..e92057b743 100644 --- a/demos/multi-template/README.md +++ b/demos/multi-template/README.md @@ -10,17 +10,15 @@ or namespace. ## Prerequisites - A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) - or [GKE Quickstart](../../README.md#gke-quickstart-development). -- `ko` installed for building images. -- A GCS bucket for storing snapshots (configured via `BUCKET_NAME` env var). + (with `ATE_INSTALL_KIND=true` exported in your shell) or [GKE Quickstart](../../README.md#gke-quickstart-development). ## How to Run on Agent Substrate ### 1. Build and Deploy > [!NOTE] -> Do not manually edit the `demos/multi-template/*.yaml.tmpl` manifests. The -> installation script automatically injects your `${BUCKET_NAME}` environment +> Do not manually edit the `demos/multi-template/*.yaml.tmpl` manifests. +> `ate-setup` automatically injects your `${BUCKET_NAME}` environment > variable during deployment. ```bash diff --git a/demos/parking/README.md b/demos/parking/README.md index fc0aa5c6b4..74b5c26487 100644 --- a/demos/parking/README.md +++ b/demos/parking/README.md @@ -14,17 +14,15 @@ See [docs/request-parking.md](../../docs/request-parking.md) for the design. ## Prerequisites - A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) - or [GKE Quickstart](../../README.md#gke-quickstart-development). -- `ko` installed for building images. -- A GCS bucket for storing snapshots (configured via `BUCKET_NAME` env var). + (with `ATE_INSTALL_KIND=true` exported in your shell) or [GKE Quickstart](../../README.md#gke-quickstart-development). ## How to Run on Agent Substrate ### 1. Build and Deploy > [!NOTE] -> Do not manually edit the `demos/parking/*.yaml.tmpl` manifests. The installation -> script automatically injects your `${BUCKET_NAME}` environment variable during +> Do not manually edit the `demos/parking/*.yaml.tmpl` manifests. `ate-setup` +> automatically injects your `${BUCKET_NAME}` environment variable during > deployment. ```bash diff --git a/demos/sandbox/README.md b/demos/sandbox/README.md index 843bef752e..b4d87af47c 100644 --- a/demos/sandbox/README.md +++ b/demos/sandbox/README.md @@ -15,9 +15,7 @@ It allows you to run arbitrary commands in an sandboxed, isolated container (run ## Prerequisites - A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) - or [GKE Quickstart](../../README.md#gke-quickstart-development). -- `ko` installed for building images. -- A GCS bucket for storing snapshots (configured in `demos/sandbox/sandbox-template.yaml.tmpl`). + (with `ATE_INSTALL_KIND=true` exported in your shell) or [GKE Quickstart](../../README.md#gke-quickstart-development). - `kubectl-ate` CLI installed (can be installed via `go install ./cmd/kubectl-ate`). ## How to Run on Agent Substrate @@ -25,9 +23,9 @@ It allows you to run arbitrary commands in an sandboxed, isolated container (run ### 1. Build and Deploy > [!NOTE] -> Do not manually edit the `demos/sandbox/*.yaml.tmpl` manifests. The installation script automatically injects your `${BUCKET_NAME}` environment variable during deployment. +> Do not manually edit the `demos/sandbox/*.yaml.tmpl` manifests. `ate-setup` automatically injects your `${BUCKET_NAME}` environment variable during deployment. -Use the core installation script to build the image and deploy the demo to your cluster: +Use `ate-setup` to build the image and deploy the demo to your cluster: ```bash go run ./cmd/ate-setup deploy demo sandbox diff --git a/docs/authorization.md b/docs/authorization.md index ab5cb19cd5..98ac56ff02 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -70,7 +70,7 @@ if those break. For example, with a JWT provider whose tokens carry the user's email as `sub`: ```sh -ATE_API_AUTHZ_BOOTSTRAP_OWNERS=alice@example.com,bob@example.com hack/install-ate.sh --deploy-ate-system +ATE_API_AUTHZ_BOOTSTRAP_OWNERS=alice@example.com,bob@example.com go run ./cmd/ate-setup deploy ate-system ``` To make your Google account a bootstrap owner, use its numeric `sub`: @@ -78,7 +78,7 @@ To make your Google account a bootstrap owner, use its numeric `sub`: ```sh gcloud auth print-identity-token | cut -d. -f2 | base64 -d 2>/dev/null | jq -r .sub # your sub ATE_API_AUTHZ_BOOTSTRAP_OWNERS=,system:serviceaccount:ate-system:ate-client \ - hack/install-ate.sh --deploy-ate-system # keep ate-client for demos and benchmarks + go run ./cmd/ate-setup deploy ate-system # keep ate-client for demos and benchmarks ``` Then add the Google provider to the `ate-api-authentication` ConfigMap and @@ -280,7 +280,7 @@ redeploy `ate-api-server` can recover: redeploy with a bootstrap owner they can use, fix the policies, then redeploy with the intended owners. ```sh -ATE_API_AUTHZ_BOOTSTRAP_OWNERS=alice@example.com hack/install-ate.sh --deploy-ate-system +ATE_API_AUTHZ_BOOTSTRAP_OWNERS=alice@example.com go run ./cmd/ate-setup deploy ate-system ``` ## Production readiness diff --git a/docs/dev/best-practices/otel-collector.md b/docs/dev/best-practices/otel-collector.md index ab34989c9d..48900738c1 100644 --- a/docs/dev/best-practices/otel-collector.md +++ b/docs/dev/best-practices/otel-collector.md @@ -14,7 +14,7 @@ shows up, see [Actor Observability](../../observability.md). |---|---|---| | [GKE Managed OpenTelemetry](#gke-managed-opentelemetry-gke-only) | Deployment behind a ClusterIP Service | GKE | | [Self-managed DaemonSet](#self-managed-daemonset) | One collector per node | You | -| [kind](#local-development-kind) | Single-replica Deployment plus Jaeger | Bundled with `install-ate-kind.sh` | +| [kind](#local-development-kind) | Single-replica Deployment plus Jaeger | Bundled with `ate-setup --kind` | This guide does not recommend one over another — that depends on your cluster, your telemetry volume, and who is on call for the collector. It documents what @@ -504,7 +504,7 @@ failure. ## Local Development (kind) -`hack/install-ate-kind.sh` installs a collector plus Jaeger into +`ate-setup --kind` installs a collector plus Jaeger into the `otel-system` namespace — see `manifests/ate-install/kind/otel-collector.yaml`. It is a single-replica Deployment, which is appropriate for a one-node kind cluster. The kind diff --git a/docs/dev/mitm-egress-test-runbook.md b/docs/dev/mitm-egress-test-runbook.md index 2d3a7c978c..2c0a54fc9f 100644 --- a/docs/dev/mitm-egress-test-runbook.md +++ b/docs/dev/mitm-egress-test-runbook.md @@ -22,7 +22,7 @@ setup in [Running the microVM runtime locally](microvm-local.md). ```bash ./hack/create-kind-cluster.sh kubectl config use-context kind-kind -ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' ./hack/install-ate-kind.sh --deploy-ate-system +ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' go run ./cmd/ate-setup --kind deploy ate-system ``` For microVM, install its assets before deploying the fixture: @@ -36,8 +36,8 @@ Deploy both egress ActorTemplates and add local-origin trust to the current gateway: ```bash -ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' ./hack/install-ate-kind.sh --deploy-demo-egress -ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' ./hack/install-ate-kind.sh --deploy-demo-egress-microvm +ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' go run ./cmd/ate-setup --kind deploy demo egress +ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' go run ./cmd/ate-setup --kind deploy demo egress-microvm ./hack/setup-e2e-egress-tls-kind.sh ``` diff --git a/docs/egress-credential-injection.md b/docs/egress-credential-injection.md index 4891b452c8..0b583ebd09 100644 --- a/docs/egress-credential-injection.md +++ b/docs/egress-credential-injection.md @@ -180,12 +180,12 @@ the credential. ### Enable it **1. The gateway.** The provider is selected when the egress gateway is -deployed, with `--deploy-ate-system` or `--deploy-atenet`, for either the Envoy +deployed, with `deploy ate-system` or `deploy atenet`, for either the Envoy or agentgateway dataplane. The selection is required: an install that wants no injection says so with `{"enabled":false}`. ```bash -hack/install-ate.sh --deploy-atenet --credential-provider='{"name":"k8s.io"}' +go run ./cmd/ate-setup deploy atenet --credential-provider='{"name":"k8s.io"}' ``` `--credential-provider` (or `ATE_CREDENTIAL_PROVIDER` when the flag is diff --git a/docs/examples/substrate-install.yaml b/docs/examples/substrate-install.yaml index a2ca9618da..5ffbca089c 100644 --- a/docs/examples/substrate-install.yaml +++ b/docs/examples/substrate-install.yaml @@ -16,7 +16,7 @@ # An Agent Substrate installation, as a document an operator owns. # # ate-setup deploy ate-system --config substrate-install.yaml -# ATE_CONFIG=substrate-install.yaml hack/install-ate.sh --deploy-ate-system +# ATE_CONFIG=substrate-install.yaml ate-setup deploy ate-system # # Every setting here can equally be an environment variable or a flag; see # docs/operator-install.md for the full list and the precedence between them. diff --git a/docs/operator-install.md b/docs/operator-install.md index 15b649a8a3..225610d191 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -23,13 +23,9 @@ Pass it with `--config`, or name it in `ATE_CONFIG`: ```bash ate-setup deploy ate-system --config substrate-install.yaml -ATE_CONFIG=substrate-install.yaml hack/install-ate.sh --deploy-ate-system +ATE_CONFIG=substrate-install.yaml ate-setup deploy ate-system ``` -`ATE_CONFIG` is how the document reaches the shell installer, which forwards -only a fixed set of flags and otherwise passes settings by inheriting the -environment. - A worked example is at [docs/examples/substrate-install.yaml](examples/substrate-install.yaml). ### Keys are nested; the table below is flat diff --git a/hack/install-ate-kind.sh b/hack/install-ate-kind.sh deleted file mode 100755 index 4c4e1ffc6e..0000000000 --- a/hack/install-ate-kind.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -o nounset -o pipefail - -ROOT="$(git rev-parse --show-toplevel)" -cd "${ROOT}" - -# shellcheck disable=SC2155 # safe initialization -goarch=$(go env GOARCH) - -# override reading dev env -export NO_DEV_ENV="true" -# we will push images to the local registry -export KO_DOCKER_REPO="${KO_DOCKER_REPO:-localhost:5001}" -# we want to build for the host architecture -export KO_DEFAULTPLATFORMS="linux/${goarch}" -# install resolved manifests using Kustomize overlay for local Kind cluster -export ATE_INSTALL_KIND="true" -# use default bucket name for local deployment -export BUCKET_NAME="ate-snapshots" -# target the local kind cluster's context (mirrors run-microvm-demo-kind.sh) so -# the install doesn't land on whatever kubeconfig current-context happens to be, -# or on nothing at all, which kubectl reports as a localhost:8080 dial failure. -# An explicit KUBECTL_CONTEXT still wins. -KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" -export KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-kind-${KIND_CLUSTER_NAME}}" -# unset other env from ate-dev-env.sh in case the developer already sourced them -unset GCE_REGION CLUSTER_LOCATION NETWORK SUBNETWORK MEMORYSTORE_INSTANCE PROJECT_ID - -go run ./cmd/ate-setup "$@" diff --git a/hack/install-ate.sh b/hack/install-ate.sh deleted file mode 100755 index edc0fb905f..0000000000 --- a/hack/install-ate.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Installs Agent Substrate. -# -# This is a translation shim: cmd/ate-setup carries out every action, and this -# script only maps the flags and environment variables the installer has always -# accepted onto its commands, so existing command lines and CI jobs keep -# working unchanged. It deliberately holds no install logic of its own. -# -# New work belongs in cmd/ate-setup. See cmd/ate-setup/commands.md for the -# equivalent invocation of each flag below, and cmd/ate-setup/cli-diff.md for -# the behaviors that differ after the move. -# -# The environment variables the installer reads -- BUCKET_NAME, KO_DOCKER_REPO, -# PROJECT_ID, CLUSTER_NAME, CLUSTER_LOCATION, KUBECTL_CONTEXT, NO_DEV_ENV, the -# ATE_API_POSTGRES_* set, and the rest -- are read by ate-setup directly, as is -# .ate-dev-env.sh. They need no translation and are not repeated here. - -set -o errexit -o nounset -o pipefail - -go run ./cmd/ate-setup "$@" diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index acfe705d23..312da2e346 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -15,7 +15,7 @@ # limitations under the License. # Install (or delete) the cluster-wide micro-VM (kata + cloud-hypervisor) -# dependencies. This is opt-in on top of hack/install-ate.sh --deploy-ate-system, +# dependencies. This is opt-in on top of ate-setup deploy ate-system, # which only installs the default gVisor SandboxConfig. Used by: # * hack/run-microvm-demo.sh (before applying the counter-microvm demo) # * benchmarking/automation/orchestrator.py (before deploying a microvm @@ -106,7 +106,7 @@ if [[ -z "${KUBECTL_CONTEXT}" ]] && ! kubectl config current-context >/dev/null exit 1 fi -# ANSI color codes for prettier output (mirrors hack/install-ate.sh). +# ANSI color codes for prettier output. COLOR_CYAN='\033[1;36m' COLOR_RESET='\033[0m' log() { diff --git a/hack/microvm-assets/README.md b/hack/microvm-assets/README.md index 1436ef4e68..00580b90d5 100644 --- a/hack/microvm-assets/README.md +++ b/hack/microvm-assets/README.md @@ -31,8 +31,9 @@ it as a device, which is what places microVM workers there. 2. **Bring up the cluster + control plane:** ```sh - hack/create-kind-cluster.sh # mounts /dev/kvm into the nodes - hack/install-ate-kind.sh # control plane + rustfs (bucket: ate-snapshots) + hack/create-kind-cluster.sh # mounts /dev/kvm into the nodes + # control plane + rustfs (bucket: ate-snapshots) + go run ./cmd/ate-setup --kind deploy ate-system --credential-provider='{"name":"k8s.io"}' ``` 3. **Stage assets into rustfs:** diff --git a/hack/microvm-assets/stage-to-rustfs.sh b/hack/microvm-assets/stage-to-rustfs.sh index 5fbd9c1e8b..4409937ec0 100755 --- a/hack/microvm-assets/stage-to-rustfs.sh +++ b/hack/microvm-assets/stage-to-rustfs.sh @@ -19,7 +19,7 @@ # plugin (per manifests/microvm/sandboxconfig-microvm.yaml.tmpl). The plugin # refuses assets stored under a snapshot location # (/atespaces/), so keep them outside it. -# Run after the cluster is up (hack/install-ate-kind.sh) and assemble.sh has produced $OUT. +# Run after the cluster is up (ate-setup --kind deploy ate-system) and assemble.sh has produced $OUT. # # The S3 client runs in a throwaway container (the same pinned amazon/aws-cli # image as the rustfs-bucket-init Job), so developers don't need the `aws` CLI diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index 5585228b9d..d532bdbe97 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -22,8 +22,8 @@ cd "${ROOT}" # Runs the E2E tests against a local Kind cluster. # # This wraps hack/run-e2e.sh with the same Kind-specific environment that -# hack/install-ate-kind.sh uses, so the snapshot bucket and image repo match -# what was installed in the cluster. All arguments are forwarded to +# ate-setup --kind uses, so the snapshot bucket and image repo match what was +# installed in the cluster. All arguments are forwarded to # hack/run-e2e.sh (run with -h to see them). # override reading dev env -- otherwise .ate-dev-env.sh would clobber the @@ -32,7 +32,7 @@ cd "${ROOT}" export NO_DEV_ENV="true" # images are pushed to the local registry export KO_DOCKER_REPO="${KO_DOCKER_REPO:-localhost:5001}" -# only build fixtures for the host arch, like install-ate-kind.sh +# only build fixtures for the host arch, like ate-setup --kind export KO_DEFAULTPLATFORMS="${KO_DEFAULTPLATFORMS:-linux/$(go env GOARCH)}" # default bucket name for local deployment (served by in-cluster rustfs) export BUCKET_NAME="ate-snapshots" diff --git a/hack/run-microvm-demo-kind.sh b/hack/run-microvm-demo-kind.sh index 683e6d8d00..0702d85fc2 100755 --- a/hack/run-microvm-demo-kind.sh +++ b/hack/run-microvm-demo-kind.sh @@ -21,7 +21,7 @@ cd "${ROOT}" # Bring up the counter-microvm demo on a local kind cluster. Wraps # hack/run-microvm-demo.sh with the same kind-specific environment that -# hack/install-ate-kind.sh uses, so the image repo and snapshot bucket match what +# ate-setup --kind uses, so the image repo and snapshot bucket match what # gets installed in the cluster. All arguments are forwarded to run-microvm-demo.sh. # shellcheck disable=SC2155 # safe initialization @@ -34,7 +34,7 @@ export NO_DEV_ENV="true" export KO_DOCKER_REPO="${KO_DOCKER_REPO:-localhost:5001}" # build for the host architecture export KO_DEFAULTPLATFORMS="linux/${goarch}" -# use the kind control-plane path (install-ate-kind.sh) + stage assets to rustfs +# use the kind control-plane path (ate-setup --kind) + stage assets to rustfs export ATE_INSTALL_KIND="true" # default bucket name for local deployment (served by the in-cluster rustfs) export BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 85baf6c4ea..9ef7334c14 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -16,10 +16,10 @@ # One-shot bring-up of the counter-microvm demo. GKE/dev-env by default; for a # local kind cluster use hack/run-microvm-demo-kind.sh (which sets the kind env -# and calls this script), mirroring install-ate.sh / install-ate-kind.sh. +# and calls this script). # # Composes: -# 1. hack/install-ate.sh --deploy-ate-system (control plane) +# 1. ate-setup deploy ate-system (control plane) # 2. hack/install-microvm-deps.sh --install (asset build/stage + cluster-wide # microvm SandboxConfig) # 3. Deploy the counter-microvm demo (worker pool manifest, atespace, and @@ -29,7 +29,7 @@ # registry / bucket settings unless NO_DEV_ENV is set. The control plane deploy # requires a credential provider selection, which reaches it only through the # environment here: export ATE_CREDENTIAL_PROVIDER (e.g. '{"name":"k8s.io"}'; -# see hack/install-ate.sh --help) before running this, the kind wrapper +# see go run ./cmd/ate-setup --help) before running this, the kind wrapper # included. The printed `kubectl ate proxy actor` step also needs # ATE_INGRESS_ALLOWED_SPIFFE_IDS to include spiffe://cluster.local/ns/ate-system/sa/atenet-client. # @@ -42,8 +42,8 @@ # PROJECT_ID (optional) GCP project for the GCS asset upload (GKE path). # ARCH target arch (default: from KO_DEFAULTPLATFORMS, else host arch). # OUT asset dir (default: $PWD/bin/microvm-assets/$ARCH, gitignored). -# ATE_INSTALL_KIND "true" for the kind path (stage assets to rustfs + install-ate-kind.sh); -# default false uploads assets to GCS + uses install-ate.sh. +# ATE_INSTALL_KIND "true" for the kind path (stage assets to rustfs + ate-setup --kind); +# default false uploads assets to GCS. set -o errexit -o nounset -o pipefail @@ -73,7 +73,7 @@ if [[ -z "${KO_DOCKER_REPO}" ]]; then fi export KO_DOCKER_REPO -# ANSI color codes for prettier output (mirrors hack/install-ate.sh). +# ANSI color codes for prettier output. COLOR_CYAN='\033[1;36m' COLOR_RESET='\033[0m' log() { @@ -82,13 +82,9 @@ log() { # --- 1. deploy the control plane ------------------------------------------- log "Deploying the ate control plane (deploy ate-system)..." -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh deploy ate-system -else - # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh deploy ate-system -fi +# ate-setup reads ATE_INSTALL_KIND from the environment, so this covers the kind +# path too. +KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" go run ./cmd/ate-setup deploy ate-system # --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- # install-microvm-deps.sh handles the assemble/stage/apply flow. Ordering @@ -103,14 +99,9 @@ if [[ -n "${KUBECTL_CONTEXT}" ]]; then fi # The demo handler applies the worker pool, creates the atespace and the -# ActorTemplate through the ate API, and waits for the golden snapshot; -# dispatch through install-ate.sh like step 1. -log "Deploying the counter-microvm demo (--deploy-demo-counter-microvm)..." -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh deploy demo counter-microvm -else - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh deploy demo counter-microvm -fi +# ActorTemplate through the ate API, and waits for the golden snapshot. +log "Deploying the counter-microvm demo (deploy demo counter-microvm)..." +KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" go run ./cmd/ate-setup deploy demo counter-microvm log "Demo applied. Next steps:" cat </dev/null 2>&1 || true -# Also delete our custom service and SC -kubectl delete service csi-hostpath-controller -n default >/dev/null 2>&1 || true -kubectl delete storageclass csi-hostpath-sc >/dev/null 2>&1 || true - -# Also clean up the host directories inside Kind node (best effort) -echo "Cleaning up CSI directories on Kind node..." -KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" -KIND_NODE=$(kubectl get nodes -l '!node-role.kubernetes.io/control-plane' -o name | head -n 1 | cut -d'/' -f2) -if [ -z "${KIND_NODE}" ]; then - KIND_NODE=$(kubectl get nodes -o name | head -n 1 | cut -d'/' -f2) -fi -echo "Using Kind node: ${KIND_NODE}" -if docker ps | grep -q "${KIND_NODE}"; then - # Unmount any stale mounts to prevent "device or resource busy" - docker exec "${KIND_NODE}" sh -c ' - for mnt in $(mount | grep /var/lib/ate | awk "{print \$3}"); do - echo "Unmounting stale mount: ${mnt}" - umount -f "${mnt}" || true - done - ' || true - # Only delete contents, keep the directory itself to preserve mounts of running pods (like atelet) - docker exec "${KIND_NODE}" sh -c 'rm -rf /var/lib/ate/*' || true -else - echo "Warning: Kind node ${KIND_NODE} not running. Skipping directory cleanup." -fi - -# 2 Expose CSI Controller over TCP Service. -# This Service must be created before the driver pod deploys for the servicednssigner -# because servicednssigner to issue a certificate. -echo "Exposing CSI Controller over TCP Service..." -cat </dev/null 2>&1; then - echo "Error: csi-hostpathplugin StatefulSet was not created!" - exit 1 -fi - -# 4. Patch the CSI Driver to mount Substrate directory -echo "Patching CSI Hostpath StatefulSet..." -kubectl patch statefulset csi-hostpathplugin -n default --patch " -spec: - template: - spec: - nodeSelector: - kubernetes.io/hostname: ${KIND_NODE} - containers: - - name: hostpath - volumeMounts: - - name: ate-base - mountPath: /var/lib/ate - mountPropagation: Bidirectional - volumes: - - name: ate-base - hostPath: - path: /var/lib/ate - type: DirectoryOrCreate -" - -# 6. Create the StorageClass -echo "Creating csi-hostpath-sc StorageClass..." -kubectl apply -f "${DRIVER_DIR}/examples/csi-storageclass.yaml" - -# 7. Wait for pods to be ready -echo "Waiting for CSI Hostpath and Proxy pods to be Ready..." -kubectl rollout status statefulset/csi-hostpathplugin -n default --timeout=120s -kubectl rollout status statefulset/csi-hostpath-socat -n default --timeout=120s - -# 8. Create the CSIDriverConfig for Substrate -echo "Creating hostpath.csi.k8s.io CSIDriverConfig..." -cat </dev/null 2>&1 || true - -echo "CSI Hostpath setup complete!" diff --git a/hack/third_party/csi-driver-nfs/deploy/example/storageclass-nfs.yaml b/hack/third_party/csi-driver-nfs/deploy/example/storageclass-nfs.yaml index e097d86bda..8d4a456ce6 100644 --- a/hack/third_party/csi-driver-nfs/deploy/example/storageclass-nfs.yaml +++ b/hack/third_party/csi-driver-nfs/deploy/example/storageclass-nfs.yaml @@ -1,6 +1,6 @@ # The csi-nfs-sc StorageClass, backed by the dev NFS server in -# hack/csi/nfs-server.yaml. `ate-setup setup csi nfs` (hack/install-ate.sh -# --setup-csi=nfs) and hack/setup-csi-nfs-kind.sh both apply this file, so the +# hack/csi/nfs-server.yaml. `ate-setup setup csi nfs` and +# hack/setup-csi-nfs-kind.sh both apply this file, so the # e2e tests, demos and benchmarks all provision from the same class. # # Based on deploy/example/storageclass-nfs.yaml from diff --git a/hack/verify-atenet-drain.sh b/hack/verify-atenet-drain.sh index f7dabd00bd..122bdd781c 100755 --- a/hack/verify-atenet-drain.sh +++ b/hack/verify-atenet-drain.sh @@ -33,7 +33,8 @@ # the e2e runner executes suites in parallel, so this check must run alone. # # Prerequisites: a cluster with the ate system and the counter demo installed -# (hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' --deploy-demo-counter), +# (go run ./cmd/ate-setup --kind deploy ate-system --credential-provider='{"name":"k8s.io"}', then +# go run ./cmd/ate-setup --kind deploy demo counter), # and no actors currently running on the counter demo pool (the pool is scaled # to 1 for the duration of the check and restored afterwards). # diff --git a/hack/verify-egress-demo.sh b/hack/verify-egress-demo.sh index 55cfb78b03..72729f62c2 100755 --- a/hack/verify-egress-demo.sh +++ b/hack/verify-egress-demo.sh @@ -16,7 +16,7 @@ # # Preconditions: # hack/create-kind-cluster.sh -# hack/install-ate-kind.sh --deploy-demo-egress, on an install whose +# go run ./cmd/ate-setup --kind deploy demo egress, on an install whose # --ingress-allowed-spiffe-ids includes spiffe://cluster.local/ns/ate-system/sa/atenet-client # # The egress demo Actor accepts {"url":"..."} and performs an HTTP GET. With diff --git a/internal/e2e/README.md b/internal/e2e/README.md index 01bdaf3e36..a3f5972267 100644 --- a/internal/e2e/README.md +++ b/internal/e2e/README.md @@ -20,8 +20,8 @@ $ go test -v ./internal/e2e/suites/... -args --e2e The e2e tests assume you have a cluster set up with Agent Substrate installed with the `--e2e-tuning` component, for example via -`hack/install-ate.sh deploy ate-system --e2e-tuning --credential-provider='{"name":"k8s.io"}'` -or `hack/install-ate-kind.sh deploy ate-system --e2e-tuning --credential-provider='{"name":"k8s.io"}'`. +`go run ./cmd/ate-setup deploy ate-system --e2e-tuning --credential-provider='{"name":"k8s.io"}'` +(add `--kind` for a local Kind cluster). The egress credential injection suite needs that bundled provider, which is what CI installs on the envoy lane; see `internal/e2e/suites/egresscredinject`. @@ -34,7 +34,7 @@ selects which, by repointing every fixture at its variant --- see [sandbox.go](sandbox.go). Unset means gVisor. ```shell -# gVisor (the default), against the demos install-ate-kind.sh deploys +# gVisor (the default), against the counter and egress demos $ hack/run-e2e-kind.sh -v -args --no-color # microVM, against the counter-microvm and egress-microvm demos @@ -45,8 +45,8 @@ The microVM lane needs its fixtures installed first, which also needs a node with `/dev/kvm` (`hack/create-kind-cluster.sh` detects one and labels the node): ```shell -$ hack/run-microvm-demo-kind.sh # counter-microvm + assets -$ hack/install-ate-kind.sh --deploy-demo-egress-microvm # egress-microvm +$ hack/run-microvm-demo-kind.sh # counter-microvm + assets +$ go run ./cmd/ate-setup --kind deploy demo egress-microvm # egress-microvm ``` A handful of knobs override the class defaults, mostly for a cluster that diff --git a/internal/e2e/credprovider.go b/internal/e2e/credprovider.go index bd18c9d819..d222a9d2c4 100644 --- a/internal/e2e/credprovider.go +++ b/internal/e2e/credprovider.go @@ -59,7 +59,7 @@ const ( // // The provider itself and the gateway's side of the connection — the // credential-provider configuration — are the install's -// (hack/install-ate.sh --credential-provider='{"name":"k8s.io"}'), not +// (ate-setup --credential-provider='{"name":"k8s.io"}'), not // something this helper can retrofit: the rollout wait below fails on a // cluster installed without them. func ConfigureCredentialProvider(t *testing.T) { diff --git a/internal/e2e/env.go b/internal/e2e/env.go index 0491442528..fa5fe5619c 100644 --- a/internal/e2e/env.go +++ b/internal/e2e/env.go @@ -36,8 +36,8 @@ func CheckEnv(keys ...string) (map[string]string, error) { } // SystemNamespaceEnv names the namespace the substrate control plane under test -// was installed into. It mirrors the ATE_NAMESPACE hack/install-ate.sh -// installed with, and the namespace the deployment was installed into. +// was installed into. It mirrors the ate-setup --namespace (ATE_NAMESPACE) +// the control plane was installed with. const SystemNamespaceEnv = "E2E_SYSTEM_NAMESPACE" // SystemNamespace returns the namespace the control plane under test runs in, diff --git a/internal/e2e/sandbox.go b/internal/e2e/sandbox.go index 878c155bc0..387e409058 100644 --- a/internal/e2e/sandbox.go +++ b/internal/e2e/sandbox.go @@ -44,7 +44,7 @@ func IsMicroVM() bool { return SandboxClass() == SandboxClassMicroVM } type Fixture struct { Namespace string Name string - // DeployWith is the install flag or script that creates the fixture, so a + // DeployWith is the ate-setup command that creates the fixture, so a // missing one reports how to fix it rather than just failing. DeployWith string } @@ -61,7 +61,7 @@ type SubstrateFixture struct { // PoolNamespace and PoolName locate the WorkerPool CRD. PoolNamespace string PoolName string - // DeployWith is the install flag or script that creates the fixture, so a + // DeployWith is the ate-setup command that creates the fixture, so a // missing one reports how to fix it rather than just failing. DeployWith string } @@ -77,7 +77,7 @@ func SubstrateCounterFixture() SubstrateFixture { Name: "counter", PoolNamespace: "ate-demo-counter", PoolName: "counter", - DeployWith: "hack/install-ate-kind.sh --deploy-demo-counter", + DeployWith: "go run ./cmd/ate-setup --kind deploy demo counter", } if IsMicroVM() { f = SubstrateFixture{ @@ -85,7 +85,7 @@ func SubstrateCounterFixture() SubstrateFixture { Name: "counter-microvm", PoolNamespace: "ate-demo-counter-microvm", PoolName: "counter-microvm", - DeployWith: "hack/install-ate-kind.sh --deploy-demo-counter-microvm", + DeployWith: "go run ./cmd/ate-setup --kind deploy demo counter-microvm", } } if v := os.Getenv("E2E_SUBSTRATE_TEMPLATE_ATESPACE"); v != "" { @@ -110,13 +110,13 @@ func EgressFixture() Fixture { return Fixture{ Namespace: "ate-demo-egress-microvm", Name: "egress-microvm", - DeployWith: "hack/install-ate-kind.sh --deploy-demo-egress-microvm", + DeployWith: "go run ./cmd/ate-setup --kind deploy demo egress-microvm", } } return Fixture{ Namespace: "ate-demo-egress", Name: "egress", - DeployWith: "hack/install-ate-kind.sh --deploy-demo-egress", + DeployWith: "go run ./cmd/ate-setup --kind deploy demo egress", } } diff --git a/internal/e2e/sandbox_test.go b/internal/e2e/sandbox_test.go index 2b4a7db7d6..9ac345e2be 100644 --- a/internal/e2e/sandbox_test.go +++ b/internal/e2e/sandbox_test.go @@ -231,7 +231,7 @@ func TestSubstrateCounterFixture(t *testing.T) { Name: "counter", PoolNamespace: "ate-demo-counter", PoolName: "counter", - DeployWith: "hack/install-ate-kind.sh --deploy-demo-counter", + DeployWith: "go run ./cmd/ate-setup --kind deploy demo counter", } if got != want { t.Errorf("SubstrateCounterFixture() = %+v, want %+v", got, want) @@ -245,7 +245,7 @@ func TestSubstrateCounterFixture(t *testing.T) { Name: "counter-microvm", PoolNamespace: "ate-demo-counter-microvm", PoolName: "counter-microvm", - DeployWith: "hack/install-ate-kind.sh --deploy-demo-counter-microvm", + DeployWith: "go run ./cmd/ate-setup --kind deploy demo counter-microvm", } if got != want { t.Errorf("SubstrateCounterFixture() = %+v, want %+v", got, want) diff --git a/internal/e2e/suites/egresscredinject/egresscredinject_test.go b/internal/e2e/suites/egresscredinject/egresscredinject_test.go index f557743730..ae64812a12 100644 --- a/internal/e2e/suites/egresscredinject/egresscredinject_test.go +++ b/internal/e2e/suites/egresscredinject/egresscredinject_test.go @@ -107,7 +107,7 @@ var withPlaceholder = []string{ // be made with the bundled provider, which deploys the k8s-credential-provider // and points the egress gateway at it. Locally: // -// hack/install-ate-kind.sh --deploy-atenet --credential-provider='{"name":"k8s.io"}' +// go run ./cmd/ate-setup --kind deploy atenet --credential-provider='{"name":"k8s.io"}' // hack/setup-e2e-egress-tls-kind.sh // hack/run-e2e-kind.sh ./internal/e2e/suites/egresscredinject -v -args --no-color func TestActorEgressCredentialInjection(t *testing.T) { diff --git a/internal/e2e/suites/egressmitm/goldenboot_test.go b/internal/e2e/suites/egressmitm/goldenboot_test.go index fb8a862500..09215bc123 100644 --- a/internal/e2e/suites/egressmitm/goldenboot_test.go +++ b/internal/e2e/suites/egressmitm/goldenboot_test.go @@ -41,7 +41,7 @@ var bootFetchManifests = e2e.SubstrateFixtureManifests{ // goldenEgressPolicy is a preview feature: install the cluster with its gate // and pass the gate to the suite, which skips the test otherwise. // -// ATE_PREVIEW=GoldenEgressPolicy hack/install-ate-kind.sh +// ATE_PREVIEW=GoldenEgressPolicy go run ./cmd/ate-setup --kind deploy ate-system --credential-provider='{"name":"k8s.io"}' // hack/run-e2e-kind.sh ./internal/e2e/suites/egressmitm -run TestGoldenBootEgress -v -args --no-color --preview=GoldenEgressPolicy func TestGoldenBootEgress(t *testing.T) { e2e.RequirePreview(t, preview.GateGoldenEgressPolicy) diff --git a/internal/e2e/suites/security/README.md b/internal/e2e/suites/security/README.md index e41410f208..d92fa3b7f0 100644 --- a/internal/e2e/suites/security/README.md +++ b/internal/e2e/suites/security/README.md @@ -17,7 +17,7 @@ hack/create-kind-cluster.sh 2. Install the control plane (slow step — pulls ~570 MB): ``` -hack/install-ate-kind.sh --deploy-ate-system +go run ./cmd/ate-setup --kind deploy ate-system --credential-provider='{"name":"k8s.io"}' ``` Stop here — the NFS/CSI and demo steps CI runs are for other suites; this suite needs none of them. diff --git a/internal/e2e/trustbundle.go b/internal/e2e/trustbundle.go index 01f0dbc781..f0615fb3d9 100644 --- a/internal/e2e/trustbundle.go +++ b/internal/e2e/trustbundle.go @@ -43,7 +43,7 @@ const ( // EgressTrustBundleObjectName is the reconciler-owned ClusterTrustBundle. EgressTrustBundleObjectName = "egress-mitm.ate.dev:mitm:primary-bundle" - // egressCAPoolSecretName is not release-prefixed: hack/install-ate.sh + // egressCAPoolSecretName is not release-prefixed: ate-setup // creates it under this fixed name and the reconciler looks it up the same // way, so it does not follow the deployment's naming. egressCAPoolSecretName = "egress-mitm-ca-pool" diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 231c44c9a5..88a0945fd6 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -302,7 +302,7 @@ data: # policy ext_proc on every request. They also carry an # #ATE_MITM_EXTPROC_FILTER marker, with a matching # #ATE_MITM_EXTPROC_CLUSTER in the cluster list: inert comments until - # hack/install-ate.sh is run with + # ate-setup is run with # --experimental-additional-egress-extproc-service, which replaces each # with a generated block for an additional, external processor. The # marker sits below the policy filter, so that processor sees only diff --git a/manifests/ate-install/base/kustomization.yaml b/manifests/ate-install/base/kustomization.yaml index e1519c964a..765aaf937b 100644 --- a/manifests/ate-install/base/kustomization.yaml +++ b/manifests/ate-install/base/kustomization.yaml @@ -19,14 +19,13 @@ kind: Kustomization # via envFrom. The kind overlay lists its own copy of the same ConfigMap name # instead of building on this directory, so the two never collide. # -# pod-certificate-controller.yaml is deliberately absent. The installers -# (hack/install-ate.sh and cmd/ate-setup) apply it on their own before this -# bundle, through the podcert-size10 overlay on large clusters, and then set -# WORKERS_PER_SIGNER on the Deployment. Listing the base file here would -# re-apply it after those steps and revert both the overlay's flags and the -# env override. The same goes for the sandboxconfig and atenet-egress files: -# the installers apply the variant they select, so they are not listed here -# either. +# pod-certificate-controller.yaml is deliberately absent. cmd/ate-setup +# applies it on its own before this bundle, through the podcert-size10 overlay +# on large clusters, and then sets WORKERS_PER_SIGNER on the Deployment. +# Listing the base file here would re-apply it after those steps and revert +# both the overlay's flags and the env override. The same goes for the +# sandboxconfig and atenet-egress files: cmd/ate-setup applies the variant it +# selects, so they are not listed here either. resources: - ../ate-api-server.yaml - ../ate-idp-server.yaml diff --git a/manifests/ate-install/cloudsql/proxy-sidecar-patch.yaml b/manifests/ate-install/cloudsql/proxy-sidecar-patch.yaml index 5981617016..307e1fc563 100644 --- a/manifests/ate-install/cloudsql/proxy-sidecar-patch.yaml +++ b/manifests/ate-install/cloudsql/proxy-sidecar-patch.yaml @@ -13,7 +13,7 @@ # limitations under the License. # Strategic-merge patch adding a Cloud SQL Auth Proxy sidecar to -# ate-api-server. Applied by hack/install-ate.sh only when +# ate-api-server. Applied by ate-setup only when # ATE_API_POSTGRES_CLOUDSQL_INSTANCE is set. # # The proxy owns transport security (TLS 1.3 tunnel with ephemeral @@ -24,7 +24,7 @@ # 127.0.0.1:5432 in plaintext that never leaves the pod's network namespace. # # All CSQL_PROXY_* configuration comes from the ate-api-server-envvars -# ConfigMap (written by install-ate.sh); the instance connection name is the +# ConfigMap (written by ate-setup); the instance connection name is the # proxy's single positional argument, expanded by Kubernetes from the same # ConfigMap. spec: diff --git a/manifests/ate-install/kind/atelet/kustomization.yaml b/manifests/ate-install/kind/atelet/kustomization.yaml index 0616973c6b..c86c0393f6 100644 --- a/manifests/ate-install/kind/atelet/kustomization.yaml +++ b/manifests/ate-install/kind/atelet/kustomization.yaml @@ -40,7 +40,7 @@ patches: # The OTLP endpoint comes from the ate-otel-config ConfigMap the # atelet DaemonSet consumes via envFrom; the kind copy lives at # ../ate-otel-config.yaml and is applied by the parent overlay or - # by hack/install-ate.sh for a targeted redeploy. + # by ate-setup for a targeted redeploy. # kind nodes ship no credential provider plugin (the args above # leave the --image-credential-provider-* flags unset), so drop # the base manifest's host mounts instead of fabricating empty diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index c757f2362f..c7cb712c3a 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -22,10 +22,9 @@ kind: Kustomization # in the bundle and an overlay would be a duplicate resource. ate-setup applies # the ConfigMap directly for the single-component redeploys. # -# pod-certificate-controller.yaml is deliberately absent, as in ../base: the -# installers (hack/install-ate.sh and cmd/ate-setup) apply and delete it on -# their own, and re-applying the base file from this bundle would revert the -# WORKERS_PER_SIGNER override. +# pod-certificate-controller.yaml is deliberately absent, as in ../base: +# cmd/ate-setup applies and deletes it on its own, and re-applying the base +# file from this bundle would revert the WORKERS_PER_SIGNER override. resources: - ./ate-api-server - ../ate-idp-server.yaml diff --git a/manifests/ate-install/podcert-size10/kustomization.yaml b/manifests/ate-install/podcert-size10/kustomization.yaml index ec6798b6be..9869b5736e 100644 --- a/manifests/ate-install/podcert-size10/kustomization.yaml +++ b/manifests/ate-install/podcert-size10/kustomization.yaml @@ -15,7 +15,7 @@ # size10 cluster overlay for podcertificate-controller: raises the client-go # rate limits above the built-in 5 QPS / 10 burst so signing keeps up with # the request volume the size10 postgres profile enables. Selected by -# hack/install-ate.sh and cmd/ate-setup when --cluster-size=size10. +# cmd/ate-setup when --cluster-size=size10. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/manifests/ate-install/postgres-size10/postgres-config-patch.yaml b/manifests/ate-install/postgres-size10/postgres-config-patch.yaml index ba73324efb..2b04e43059 100644 --- a/manifests/ate-install/postgres-size10/postgres-config-patch.yaml +++ b/manifests/ate-install/postgres-size10/postgres-config-patch.yaml @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Merged into postgres-config when --cluster-size=size10: hack/install-ate.sh -# applies it with `kubectl patch --type merge`, cmd/ate-setup folds it into -# the rendered ConfigMap before the apply. Either way only the postgresql.conf +# Merged into postgres-config when --cluster-size=size10: cmd/ate-setup folds +# it into the rendered ConfigMap before the apply. Only the postgresql.conf # key is replaced, without touching pg_hba.conf or reload-tls.sh, both defined # in manifests/ate-install/postgres/postgres.yaml. # -# Sized to match the size10 container resources both installers set (140 GiB +# Sized to match the size10 container resources ate-setup sets (140 GiB # memory, 80 vCPU request, no CPU limit, on a dedicated node). Every value # below depends on those numbers -- if the container is resized the tuning # has to move with it. diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index 19bd5b6fd3..2423d99e03 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -14,7 +14,7 @@ # Cluster-wide SandboxConfig for the micro-VM (kata + cloud-hypervisor) sandbox # class. Unlike sandboxconfig-gvisor.yaml (applied unconditionally by -# hack/install-ate.sh --deploy-ate-system), this is opt-in: apply via +# ate-setup deploy ate-system), this is opt-in: apply via # hack/install-microvm-deps.sh --install after staging the asset set # (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh). A microvm WorkerPool # resolves it as its class default (or names it via configRef: microvm), so diff --git a/tools/setup-gcp/README.md b/tools/setup-gcp/README.md index f0c685de76..82fa35866a 100644 --- a/tools/setup-gcp/README.md +++ b/tools/setup-gcp/README.md @@ -254,7 +254,7 @@ project-level IAM permissions. atelet and ate-api authenticate via [GKE Workload Identity Federation](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity): their Kubernetes ServiceAccounts (`ate-system/atelet` and -`ate-system/ate-api-server`, created by `install-ate.sh`) are addressed +`ate-system/ate-api-server`, created by `ate-setup`) are addressed directly as IAM principals. The one Google service account `create iam` creates is the snapshot transfer account⁴, which ate-api-server's credential exchange impersonates and downscopes per actor for every snapshot transfer. diff --git a/tools/setup-gcp/cloud-sql.md b/tools/setup-gcp/cloud-sql.md index e23db311cf..ee8c6302f0 100644 --- a/tools/setup-gcp/cloud-sql.md +++ b/tools/setup-gcp/cloud-sql.md @@ -21,7 +21,7 @@ authentication**](https://docs.cloud.google.com/sql/docs/postgres/iam-authentica - **Cloud-agnostic code** — ateapi itself knows nothing about Cloud SQL. The sidecar is a deployment-time patch (`manifests/ate-install/cloudsql/proxy-sidecar-patch.yaml`) applied by - `hack/install-ate.sh` only when a Cloud SQL instance is configured. + `ate-setup` only when a Cloud SQL instance is configured. ## 1. Provision @@ -163,7 +163,7 @@ Optional environment variables: (default: `max(4, NumCPU)`). It does not affect the owner and watch pools, which are capped at 2 and 3 connections. -Changing the installed configuration rolls ate-api-server: the install script +Changing the installed configuration rolls ate-api-server: ate-setup stamps a hash of the rendered configuration into the pod template (`ate.dev/env-hash`). diff --git a/tools/setup-gcp/cmd/cloudsql.go b/tools/setup-gcp/cmd/cloudsql.go index 7c0ca565a1..8e8e43fac3 100644 --- a/tools/setup-gcp/cmd/cloudsql.go +++ b/tools/setup-gcp/cmd/cloudsql.go @@ -35,7 +35,7 @@ import ( ) // The ateapi store database and the Kubernetes identity that connects to it. -// These match the atepg store (hack/install-ate.sh) and the ate-api-server +// These match the atepg store (ate-setup) and the ate-api-server // deployment. const ( cloudSQLDatabase = "atepg" @@ -196,7 +196,7 @@ func cloudSQLInstanceSpec(cfg *Config) (*sqladmin.DatabaseInstance, error) { settings := &sqladmin.Settings{ Tier: cfg.CloudSQLTier, // This tool provisions private-services-access instances exclusively. - // The ATE_API_POSTGRES_CLOUDSQL_IP_TYPE setting in install-ate.sh exists + // The ATE_API_POSTGRES_CLOUDSQL_IP_TYPE setting in ate-setup exists // only to connect to public/PSC instances provisioned out-of-band. IpConfiguration: &sqladmin.IpConfiguration{ Ipv4Enabled: false, @@ -396,7 +396,7 @@ Cloud SQL is provisioned. Two steps remain: export ATE_API_POSTGRES_CLOUDSQL_INSTANCE=%s:%s:%s export ATE_API_POSTGRES_CLOUDSQL_GSA=%s - ./hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' + go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}' See tools/setup-gcp/cloud-sql.md for details and verification steps. `, cfg.ProjectID, cfg.Region, cfg.CloudSQLInstance, gsa) diff --git a/tools/setup-gcp/cmd/cloudsql_test.go b/tools/setup-gcp/cmd/cloudsql_test.go index 5b9a52594b..84040a7f6e 100644 --- a/tools/setup-gcp/cmd/cloudsql_test.go +++ b/tools/setup-gcp/cmd/cloudsql_test.go @@ -34,7 +34,7 @@ func TestCloudSQLIdentityDerivations(t *testing.T) { } // The IAM database username is the GSA email without the // .gserviceaccount.com suffix — this must match what operators GRANT to - // and what install-ate.sh synthesizes into the DSN. + // and what ate-setup synthesizes into the DSN. if got, want := cloudSQLDatabaseUser(cloudSQLGSAEmail(cfg)), "ate-api-server@my-proj.iam"; got != want { t.Errorf("cloudSQLDatabaseUser() = %q, want %q", got, want) } From 893a45817bee31bc8fd04ce7d7e7eab1b78897f7 Mon Sep 17 00:00:00 2001 From: haiyanmeng Date: Wed, 28 Oct 2026 21:52:04 +0000 Subject: [PATCH 42/53] Document running the ate-api-server cold resume tests from a workstation (#3394) `benchmarking/automation/README.md` only covers running tests from the CronJob. Running the cold resume tests once from a workstation takes steps that aren't written down: the node pools for `--cordon-system-infra-components`, a `file://` clone, a separate `SUBSTRATE_DIR` and `KUBECONFIG`, and the variables the install needs in the target cluster file. Step 6 sets `orchestrator.SUBSTRATE_DIR` from a small wrapper script, because the orchestrator has no flag for it. I can add a `--substrate-dir` flag in a follow-up. AI use: Claude Code helped write the guide and this description, and ran the steps on a GKE cluster. - [x] Tests pass - [x] Docs updated Signed-off-by: Haiyan Meng --- benchmarking/README.md | 3 + benchmarking/automation/README.md | 2 + benchmarking/automation/ateapi-cold-resume.md | 153 ++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 benchmarking/automation/ateapi-cold-resume.md diff --git a/benchmarking/README.md b/benchmarking/README.md index 6e9b0daf41..9c17596d03 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -569,6 +569,9 @@ deleted, and its user creates another on its next iteration. When a step lowers the user count, the actors of the stopped users stay suspended, unused, until boomer exits. +To run the cold resume tests in `automation/tests.yaml` from a workstation, +see [automation/ateapi-cold-resume.md](automation/ateapi-cold-resume.md). + #### ResumePaused Configuration Knobs * `--target-rps`: Cold resumes per second the whole boomer process paces to diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 50c5cd5d62..30a9dfe83c 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -7,6 +7,8 @@ Each run uploads results to GCS via `benchmarking/locust/runner.py` (`type: locust`) or `benchmarking/nighthawk-ingress/runner.py` (`type: nighthawk-ingress`, the router capacity benchmark — see [benchmarking/nighthawk-ingress/README.md](../nighthawk-ingress/README.md)). +To run the cold resume tests from your workstation instead, see +[ateapi-cold-resume.md](ateapi-cold-resume.md). ## How it works diff --git a/benchmarking/automation/ateapi-cold-resume.md b/benchmarking/automation/ateapi-cold-resume.md new file mode 100644 index 0000000000..805d2383c2 --- /dev/null +++ b/benchmarking/automation/ateapi-cold-resume.md @@ -0,0 +1,153 @@ +# Running the ate-api-server cold resume test locally + +The `ateapi_cold_resume_pause_fake_*` tests in [tests.yaml](tests.yaml) +measure how many cold resumes per second ate-api-server and Postgres sustain. +They run the [ResumePaused benchmark](../README.md#resumepaused-benchmark) on +the [fake data plane](../README.md#isolate-mode) as a rate ladder. + +Both tests run through `benchmarking/automation/orchestrator.py`. The +[automation README](README.md#how-it-works) covers what it does for each test +and how to run it from the CronJob. This doc covers a one-off run from your +workstation. Run the commands from the repository root. + +## 1. Cluster + +Create the test cluster as in +[Test cluster prerequisites](README.md#test-cluster-prerequisites). + +Both tests pass `--cordon-system-infra-components`, which needs the two node +pools in [System Infra Isolation](../../docs/architecture.md#system-infra-isolation): +`ate-system-infra` and `ate-postgres`. If the `ate-postgres` pool is missing, +the install fails before it applies Postgres. If the `ate-system-infra` pool +is missing or too small, its pods stay Pending and the install fails when +the rollout wait times out. Size that pool for ate-api-server's +`replicas × cpu` plus the other system infra pods. Both tests run one +ate-api-server at 2 CPUs. + +```bash +gcloud container node-pools create ate-system-infra \ + --cluster=$CLUSTER_NAME --location=$CLUSTER_LOCATION \ + --machine-type= --num-nodes=1 \ + --node-labels=ate.dev/workloadType=ate-system-infra \ + --node-taints=ate.dev/workloadType=ate-system-infra:NoSchedule + +gcloud container node-pools create ate-postgres \ + --cluster=$CLUSTER_NAME --location=$CLUSTER_LOCATION \ + --machine-type= --num-nodes=1 \ + --node-labels=ate.dev/workloadType=ate-postgres \ + --node-taints=ate.dev/workloadType=ate-postgres:NoSchedule +``` + +The default pool runs the locust runner (8 CPUs in `_2cpu`) and fake-atelet. + +## 2. Prometheus + +The orchestrator does not deploy Prometheus. The runner needs it for +`server_summary.json` and the per-step rows in `stats.jsonl`, so install it +once as in [Prometheus + Grafana](../README.md#optional-prometheus--grafana). + +## 3. Workstation + +You need `gcloud`, `kubectl`, `docker`, Go, and Python 3 with PyYAML. ko +runs from `hack/tools/ko` through `go tool`, so it need not be installed. +The orchestrator polls `docker info` first and fails if Docker is not +running within 2 minutes. + +```bash +python3 -m venv benchmarking/automation/venv +benchmarking/automation/venv/bin/pip install -r benchmarking/automation/requirements.txt +gcloud auth login +gcloud auth configure-docker gcr.io +``` + +Your account needs the orchestrator's roles, and the runner's KSA needs +write access to the results bucket. See +[IAM prerequisites](README.md#iam-prerequisites). + +## 4. Pick the tests + +The orchestrator runs every test in the file you pass to `--tests`, so copy +the entries you want into a new file: + +```bash +python3 - <<'PY' > /tmp/cold-tests.yaml +import yaml +want = {"ateapi_cold_resume_pause_fake_smoke"} +t = yaml.safe_load(open("benchmarking/automation/tests.yaml"))["tests"] +print(yaml.safe_dump({"tests": [e for e in t if e["name"] in want]}, sort_keys=False)) +PY +``` + +| Test | Load | Duration | +|---|---|---| +| `ateapi_cold_resume_pause_fake_smoke` | 50 users, 20 then 50 cold resumes/s | 6m | +| `ateapi_cold_resume_pause_fake_2cpu` | 1000 users, 100 to 1500 cold resumes/s in 7 steps | 23m | + +Run `_smoke` first to check the setup before the 23 minute ladder. The +header comments in `tests.yaml` describe the fields, such as +`controlPlane`, `isolate` and `--ladder`. If you change `replicas` or +`poolMaxConns`, keep within Postgres's connection slots as described in +[Running at scale](../README.md#running-at-scale). + +## 5. Target cluster file + +The orchestrator copies `.sh` from the `--target-cluster-dir` +directory into the clone as `.ate-dev-env.sh`. Both tests use +`targetCluster: dev`, so step 6 copies your `.ate-dev-env.sh` there as +`dev.sh`. Create it from +[hack/ate-dev-env.sh.example](../../hack/ate-dev-env.sh.example) +if you don't have one. The install needs `PROJECT_ID`, `CLUSTER_NAME`, +`CLUSTER_LOCATION`, `KO_DOCKER_REPO`, `SNAPSHOT_LOCATION`, +`SNAPSHOT_TRANSFER_GSA_NAME` and `ATE_CREDENTIAL_PROVIDER`. + +## 6. Run + +The orchestrator clones `--branch` from `--repo`, so commit your changes +first. A `file://` URL clones your local checkout. Set `SUBSTRATE_DIR` to a +directory that does not exist yet, because the default is the pod path +`/workspace/substrate`. The orchestrator runs `gcloud container clusters +get-credentials`, so point `KUBECONFIG` at a separate file to keep your +current context: + +```bash +S=$PWD # this checkout +D=$HOME/bench-runs/cold-$(date +%Y%m%d-%H%M) +mkdir -p $D/target-clusters +cp $S/.ate-dev-env.sh $D/target-clusters/dev.sh +export KUBECONFIG=$D/kubeconfig + +cat > $D/run.py </ \ + --tests /tmp/cold-tests.yaml \ + --target-cluster-dir $D/target-clusters \ + --manifests-dir $S/benchmarking/automation/manifests \ + --junit-output $D/junit.xml 2>&1 | tee $D/orchestrator.log +``` + +Keep your workstation awake for the whole run. If it suspends, the +orchestrator pauses too, and the teardown and the next test wait for it. For +long ladders, run the command under `setsid nohup` or in a VM. + +## 7. Results + +[Analyzing a ResumePaused Ladder Run](../README.md#analyzing-a-resumepaused-ladder-run) +explains the per-step rows and where the run path is in the log. +[Benchmark output files](../README.md#benchmark-output-files) +lists the files. To copy them all down: + +```bash +st=$(grep -o 'gs://[^ ]*/status.json' $D/orchestrator.log | head -1) +mkdir -p $D/results && gcloud storage cp "${st%/status.json}/*" $D/results/ +``` + +The capacity is the highest step whose `resume_rps` keeps up with its +`target_rps` and has no failures. From 34cce1e936443c94c31ef66dc259df37a7b6853e Mon Sep 17 00:00:00 2001 From: Chenyi Wang Date: Wed, 28 Oct 2026 22:37:15 +0000 Subject: [PATCH 43/53] ateworker: reuse a completed checkpoint only on the same boot (#3213) Follow-up to #3015. Part of #2836. A retried CheckpointWorkload reuses the worker's record of a completed checkpoint. The worker does not sync the snapshot files, so after a node reboot the record and the files can survive with torn contents. The retry then reused them, and atelet stamped the local snapshot with the new boot ID, so its pre-reboot check no longer catches them. - RecordCheckpoint records the node's boot ID, and ReuseCheckpoint skips a record from another boot. After a reboot, the retry fails instead of committing files that may be torn. - atelet's boot ID reader moves to internal/bootid so the workers can use it. TestRecordAndReuseCheckpoint covers a record from another boot. AI use: AI-assisted. I reviewed and tested it. - [x] Tests pass --------- Signed-off-by: Chenyi Wang --- cmd/atelet/bootid.go | 23 --------- cmd/atelet/bootid_test.go | 30 ----------- cmd/atelet/main.go | 5 +- cmd/ateworker-gvisor/main.go | 17 +++++-- cmd/ateworker-microvm/checkpoint.go | 4 +- cmd/ateworker-microvm/main.go | 13 ++++- internal/ateworker/checkpoint.go | 19 ++++--- internal/ateworker/checkpoint_test.go | 36 ++++++++----- internal/bootid/bootid.go | 41 +++++++++++++++ .../bootid}/bootid_linux_test.go | 12 ++--- internal/bootid/bootid_test.go | 51 +++++++++++++++++++ 11 files changed, 161 insertions(+), 90 deletions(-) create mode 100644 internal/bootid/bootid.go rename {cmd/atelet => internal/bootid}/bootid_linux_test.go (70%) create mode 100644 internal/bootid/bootid_test.go diff --git a/cmd/atelet/bootid.go b/cmd/atelet/bootid.go index 2f75216fce..32c9e6485d 100644 --- a/cmd/atelet/bootid.go +++ b/cmd/atelet/bootid.go @@ -15,32 +15,9 @@ package main import ( - "fmt" - "os" - "strings" - "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" ) -// bootIDPath is where Linux exposes the boot ID: a random UUID generated once -// per boot, so it changes on every reboot, clean or not, and on nothing else. -// It is not namespaced, so atelet reads the node's. -const bootIDPath = "/proc/sys/kernel/random/boot_id" - -// readBootID returns the boot ID read from path. It is never empty, so a local -// snapshot whose manifest records none never matches it. -func readBootID(path string) (string, error) { - b, err := os.ReadFile(path) - if err != nil { - return "", fmt.Errorf("while reading the boot ID: %w", err) - } - id := strings.TrimSpace(string(b)) - if id == "" { - return "", fmt.Errorf("boot ID file %s is empty", path) - } - return id, nil -} - // localSnapshotBootCheck returns the snapshot tracker's check of whether an // actor's local snapshot was written on the boot bootID: whether the // snapshot's manifest records bootID. A manifest that is missing or does not diff --git a/cmd/atelet/bootid_test.go b/cmd/atelet/bootid_test.go index 52046be46b..89c6f3c4fd 100644 --- a/cmd/atelet/bootid_test.go +++ b/cmd/atelet/bootid_test.go @@ -22,36 +22,6 @@ import ( "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" ) -func TestReadBootID(t *testing.T) { - t.Run("reads and trims boot ID", func(t *testing.T) { - p := filepath.Join(t.TempDir(), "boot_id") - const want = "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f" - if err := os.WriteFile(p, []byte(want+"\n"), 0o600); err != nil { - t.Fatal(err) - } - got, err := readBootID(p) - if err != nil || got != want { - t.Errorf("readBootID = (%q, %v), want (%q, nil)", got, err, want) - } - }) - - t.Run("rejects empty file", func(t *testing.T) { - p := filepath.Join(t.TempDir(), "boot_id") - if err := os.WriteFile(p, []byte(" \n\t"), 0o600); err != nil { - t.Fatal(err) - } - if _, err := readBootID(p); err == nil { - t.Error("readBootID accepted an empty boot ID file") - } - }) - - t.Run("rejects missing file", func(t *testing.T) { - if _, err := readBootID(filepath.Join(t.TempDir(), "missing")); err == nil { - t.Error("readBootID accepted a missing file") - } - }) -} - // localSnapshotBootCheck matches a local snapshot only when its manifest // records the given boot ID. func TestLocalSnapshotBootCheck(t *testing.T) { diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 4682bf0f00..a16b722839 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -52,6 +52,7 @@ import ( "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/atelet" + "go.ate.dev/substrate/internal/bootid" "go.ate.dev/substrate/internal/clustertrustbundle" "go.ate.dev/substrate/internal/credbundle" "go.ate.dev/substrate/internal/imagecache" @@ -384,7 +385,7 @@ func main() { defer ateapiConn.Close() workerService := ateapi.NewWorkerServiceClient(ateapiConn) - bootID, err := readBootID(bootIDPath) + bootID, err := bootid.Read(bootid.Path) if err != nil { serverboot.Fatal(ctx, "Failed to read the node's boot ID", err) } @@ -752,7 +753,7 @@ type Atelet struct { // failures, so a restore that can never succeed stops being retried (see // boundRetryableRestoreError). restoreFailures restoreFailureCounter - // bootID is the node's boot ID (see readBootID), recorded in the manifest + // bootID is the node's boot ID (see bootid.Read), recorded in the manifest // of each local snapshot. A local snapshot recording another, or none, // was written before the node last rebooted. bootID string diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index 908afccf03..dc0df74dde 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -49,6 +49,7 @@ import ( "go.ate.dev/substrate/internal/ateworker/register" "go.ate.dev/substrate/internal/ateworkervalidation" "go.ate.dev/substrate/internal/atunnel" + "go.ate.dev/substrate/internal/bootid" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/ocispec" @@ -140,6 +141,10 @@ func do(ctx context.Context) error { if *usageSampleInterval < minUsageSampleInterval { return fmt.Errorf("--usage-sample-interval must be at least %v, got %v", minUsageSampleInterval, *usageSampleInterval) } + bootID, err := bootid.Read(bootid.Path) + if err != nil { + return err + } const serviceName = "ateworker-gvisor" // Export through atelet's node-local relay when it is there, so telemetry @@ -239,7 +244,7 @@ func do(ctx context.Context) error { if err != nil { return err } - ateworkerService := NewService(tunnel, actorLogger, *maxActors, flags) + ateworkerService := NewService(tunnel, actorLogger, *maxActors, flags, bootID) ateworkerService.throttleMemory = psiEnabled(defaultCgroupRoot) if !ateworkerService.throttleMemory { slog.WarnContext(ctx, "Kernel has no cgroup memory pressure (PSI), so sandboxes keep memory.max and an OOM kill in one can restart the worker") @@ -345,6 +350,9 @@ type AteWorkerService struct { // crashReporter reports actors whose sandbox died. Nil if it could not be // set up. crashReporter crashReporter + // bootID is the node's boot ID, recorded with each completed checkpoint so + // a retry after a reboot does not reuse it. + bootID string // Guards actors and draining against concurrent stats reads. actorsMu sync.RWMutex @@ -394,7 +402,7 @@ type AteWorkerService struct { var _ ateworkerpb.WorkerServer = (*AteWorkerService)(nil) // NewService creates a new AteWorkerService. -func NewService(tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, maxActors int, flags *flagStore) *AteWorkerService { +func NewService(tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, maxActors int, flags *flagStore, bootID string) *AteWorkerService { s := &AteWorkerService{ locks: actorlock.New(), inFlight: actorlock.NewInFlight(), @@ -405,6 +413,7 @@ func NewService(tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, maxAc cgroupRoot: defaultCgroupRoot, flags: flags, checkpointWrites: make(chan struct{}, checkpointWriteSlots), + bootID: bootID, } s.borrow = &memborrow.Ledger{Unbooked: s.unbookedMemory} return s @@ -787,7 +796,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke defer s.locks.Unlock(req.GetActorUid()) if s.lookupActor(req.GetActorUid()) == nil { - if resp := ateworker.ReuseCheckpoint(req.GetActorDirs().GetCheckpointDir(), req.GetFidelity()); resp != nil { + if resp := ateworker.ReuseCheckpoint(req.GetActorDirs().GetCheckpointDir(), s.bootID, req.GetFidelity()); resp != nil { slog.InfoContext(ctx, "Reusing completed checkpoint from previous attempt", slog.String("actorUID", req.GetActorUid())) return resp, nil } @@ -980,7 +989,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke s.actorLogger.EmitLifecycleLog(ctx, "Actor checkpointed", attribution) resp := &ateworkerpb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles, DataSnapshotFiles: durableFiles} - if err := ateworker.RecordCheckpoint(checkpointPath, fidelity, resp); err != nil { + if err := ateworker.RecordCheckpoint(checkpointPath, s.bootID, fidelity, resp); err != nil { slog.WarnContext(ctx, "Failed to record checkpoint completion marker", slog.String("actorUID", req.GetActorUid()), slog.Any("err", err)) } return resp, nil diff --git a/cmd/ateworker-microvm/checkpoint.go b/cmd/ateworker-microvm/checkpoint.go index de567b50d9..a77e5b3ebe 100644 --- a/cmd/ateworker-microvm/checkpoint.go +++ b/cmd/ateworker-microvm/checkpoint.go @@ -82,7 +82,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke defer s.locks.Unlock(req.GetActorUid()) if s.lookupActor(req.GetActorUid()) == nil { - if resp := ateworker.ReuseCheckpoint(req.GetActorDirs().GetCheckpointDir(), req.GetFidelity()); resp != nil { + if resp := ateworker.ReuseCheckpoint(req.GetActorDirs().GetCheckpointDir(), s.bootID, req.GetFidelity()); resp != nil { slog.InfoContext(ctx, "Reusing completed checkpoint from previous attempt", slog.String("actorUID", req.GetActorUid())) return resp, nil } @@ -263,7 +263,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke slog.Duration("teardown", dTeardown)) s.recordFinalIfEnded(ctx, hosted) resp := &ateworkerpb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles, DataSnapshotFiles: durableFiles} - if err := ateworker.RecordCheckpoint(checkpointDir, scope, resp); err != nil { + if err := ateworker.RecordCheckpoint(checkpointDir, s.bootID, scope, resp); err != nil { slog.WarnContext(ctx, "Failed to record checkpoint completion marker", slog.String("actorUID", actorUID), slog.Any("err", err)) } return resp, nil diff --git a/cmd/ateworker-microvm/main.go b/cmd/ateworker-microvm/main.go index 475bff12f4..4a1cb8570e 100644 --- a/cmd/ateworker-microvm/main.go +++ b/cmd/ateworker-microvm/main.go @@ -49,6 +49,7 @@ import ( "go.ate.dev/substrate/internal/ateworker/pid1" "go.ate.dev/substrate/internal/ateworker/register" "go.ate.dev/substrate/internal/atunnel" + "go.ate.dev/substrate/internal/bootid" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/otlprelay" "go.ate.dev/substrate/internal/preview" @@ -124,6 +125,10 @@ func do(ctx context.Context) error { if *usageSampleInterval < minUsageSampleInterval { return fmt.Errorf("--usage-sample-interval must be at least %v, got %v", minUsageSampleInterval, *usageSampleInterval) } + bootID, err := bootid.Read(bootid.Path) + if err != nil { + return err + } const serviceName = "ateworker-microvm" // Export through atelet's node-local relay when it is there, so telemetry @@ -233,7 +238,7 @@ func do(ctx context.Context) error { if err != nil { return err } - ateworkerService := NewService(*podUID, *chBinary, *guestDebug, *maxActors, tunnel, actorLogger) + ateworkerService := NewService(*podUID, *chBinary, *guestDebug, *maxActors, tunnel, actorLogger, bootID) // The controller sets both from the downward API. pool := workerstats.Pool{Namespace: os.Getenv("POD_NAMESPACE"), Name: os.Getenv("WORKER_POOL_NAME")} if pool.Namespace == "" || pool.Name == "" { @@ -357,6 +362,9 @@ type AteWorkerService struct { // crashReporter reports actors whose micro-VM died. Nil if it could not // be set up. crashReporter crashReporter + // bootID is the node's boot ID, recorded with each completed checkpoint so + // a retry after a reboot does not reuse it. + bootID string // shuttingDown is set once SIGTERM has been received. While true, new workload // RPCs are rejected with codes.Unavailable so the control plane reschedules. @@ -402,7 +410,7 @@ type AteWorkerService struct { var _ ateworkerpb.WorkerServer = (*AteWorkerService)(nil) // NewService creates a new AteWorkerService. -func NewService(podUID, chBinary string, guestDebug bool, maxActors int, tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger) *AteWorkerService { +func NewService(podUID, chBinary string, guestDebug bool, maxActors int, tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, bootID string) *AteWorkerService { s := &AteWorkerService{ locks: actorlock.New(), inFlight: actorlock.NewInFlight(), @@ -415,6 +423,7 @@ func NewService(podUID, chBinary string, guestDebug bool, maxActors int, tunnel snapshotWrites: make(chan struct{}, snapshotWriteSlots), tunnel: tunnel, actorLogger: actorLogger, + bootID: bootID, } s.borrow = &memborrow.Ledger{Unbooked: s.unbookedMemory} return s diff --git a/internal/ateworker/checkpoint.go b/internal/ateworker/checkpoint.go index 66611a3364..69775e6fd4 100644 --- a/internal/ateworker/checkpoint.go +++ b/internal/ateworker/checkpoint.go @@ -42,16 +42,20 @@ func CheckpointContext(ctx context.Context) (context.Context, context.CancelFunc const checkpointDoneFile = ".checkpoint-done.json" type checkpointRecord struct { + // BootID is the node's boot ID when the checkpoint finished. The worker + // does not sync the snapshot files, so after a reboot they may be torn. + BootID string `json:"boot_id"` Fidelity ateworkerpb.SnapshotFidelity `json:"fidelity"` SnapshotFiles []string `json:"snapshot_files,omitempty"` DataSnapshotFiles []string `json:"data_snapshot_files,omitempty"` } // RecordCheckpoint writes the completion marker into checkpointDir so a retry -// after a caller timeout or atelet restart can reuse the finished -// CheckpointWorkloadResponse. -func RecordCheckpoint(checkpointDir string, fidelity ateworkerpb.SnapshotFidelity, resp *ateworkerpb.CheckpointWorkloadResponse) error { +// after a caller timeout or atelet restart on the boot bootID can reuse the +// finished CheckpointWorkloadResponse. +func RecordCheckpoint(checkpointDir, bootID string, fidelity ateworkerpb.SnapshotFidelity, resp *ateworkerpb.CheckpointWorkloadResponse) error { b, err := json.Marshal(checkpointRecord{ + BootID: bootID, Fidelity: fidelity, SnapshotFiles: resp.GetSnapshotFiles(), DataSnapshotFiles: resp.GetDataSnapshotFiles(), @@ -67,15 +71,16 @@ func RecordCheckpoint(checkpointDir string, fidelity ateworkerpb.SnapshotFidelit } // ReuseCheckpoint returns the recorded CheckpointWorkloadResponse from -// checkpointDir if the completion marker exists, its fidelity matches, and all -// listed snapshot files are still present on disk. -func ReuseCheckpoint(checkpointDir string, fidelity ateworkerpb.SnapshotFidelity) *ateworkerpb.CheckpointWorkloadResponse { +// checkpointDir if the completion marker exists, was recorded on the boot +// bootID, its fidelity matches, and all listed snapshot files are still present +// on disk. +func ReuseCheckpoint(checkpointDir, bootID string, fidelity ateworkerpb.SnapshotFidelity) *ateworkerpb.CheckpointWorkloadResponse { b, err := os.ReadFile(filepath.Join(checkpointDir, checkpointDoneFile)) if err != nil { return nil } var rec checkpointRecord - if err := json.Unmarshal(b, &rec); err != nil || rec.Fidelity != fidelity { + if err := json.Unmarshal(b, &rec); err != nil || rec.BootID != bootID || rec.Fidelity != fidelity { return nil } // DataSnapshotFiles is the VOLUMES subset of SnapshotFiles: the same file diff --git a/internal/ateworker/checkpoint_test.go b/internal/ateworker/checkpoint_test.go index 2ae265141e..9a3ef865a3 100644 --- a/internal/ateworker/checkpoint_test.go +++ b/internal/ateworker/checkpoint_test.go @@ -25,6 +25,9 @@ import ( "go.ate.dev/substrate/internal/proto/ateworkerpb" ) +// testBootID is the boot the tests record and reuse checkpoints on. +const testBootID = "boot-1" + func TestRecordAndReuseCheckpoint(t *testing.T) { checkpointDir := t.TempDir() imgPath := filepath.Join(checkpointDir, "checkpoint.img") @@ -34,33 +37,38 @@ func TestRecordAndReuseCheckpoint(t *testing.T) { // Without a completion marker (e.g. partial files from an interrupted attempt), // ReuseCheckpoint returns nil. - if got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil { + if got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil { t.Fatalf("ReuseCheckpoint(before RecordCheckpoint) = %v, want nil", got) } wantResp := &ateworkerpb.CheckpointWorkloadResponse{ SnapshotFiles: []string{"checkpoint.img"}, } - if err := RecordCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, wantResp); err != nil { + if err := RecordCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, wantResp); err != nil { t.Fatalf("RecordCheckpoint: %v", err) } // Reusing with matching fidelity and intact file succeeds. - got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) + got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if got == nil || !slices.Equal(got.GetSnapshotFiles(), wantResp.GetSnapshotFiles()) { t.Fatalf("ReuseCheckpoint() = %v, want %v", got, wantResp) } + // A marker recorded before the node rebooted is not reused. + if got := ReuseCheckpoint(checkpointDir, "boot-2", ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil { + t.Fatalf("ReuseCheckpoint(another boot) = %v, want nil", got) + } + // Mismatched fidelity is not reused. - if got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES); got != nil { + if got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES); got != nil { t.Fatalf("ReuseCheckpoint(different fidelity) = %v, want nil", got) } - // Once atelet moves/removes the file from checkpoint-state/, the marker is not reused. + // Once the file is gone from checkpoint-state/, the marker is not reused. if err := os.Remove(imgPath); err != nil { t.Fatal(err) } - if got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil { + if got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil { t.Fatalf("ReuseCheckpoint(after file removed) = %v, want nil", got) } } @@ -82,10 +90,10 @@ func TestReuseCheckpointDataSnapshotFilesByFidelity(t *testing.T) { SnapshotFiles: []string{"checkpoint.img", "fs/fscheckpoint.pb", "fs/multitar.img"}, DataSnapshotFiles: []string{"fs/fscheckpoint.pb", "fs/multitar.img"}, } - if err := RecordCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, memResp); err != nil { + if err := RecordCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, memResp); err != nil { t.Fatalf("RecordCheckpoint(MEMORY): %v", err) } - got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) + got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if got == nil || !slices.Equal(got.GetSnapshotFiles(), memResp.GetSnapshotFiles()) || !slices.Equal(got.GetDataSnapshotFiles(), memResp.GetDataSnapshotFiles()) { t.Fatalf("ReuseCheckpoint(MEMORY) = %v, want %v", got, memResp) } @@ -95,10 +103,10 @@ func TestReuseCheckpointDataSnapshotFilesByFidelity(t *testing.T) { SnapshotFiles: []string{"checkpoint.img"}, DataSnapshotFiles: []string{"fs/fscheckpoint.pb"}, } - if err := RecordCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, badSubset); err != nil { + if err := RecordCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, badSubset); err != nil { t.Fatalf("RecordCheckpoint(badSubset): %v", err) } - if got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil { + if got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil { t.Fatalf("ReuseCheckpoint(data file not in SnapshotFiles) = %v, want nil", got) } @@ -108,19 +116,19 @@ func TestReuseCheckpointDataSnapshotFilesByFidelity(t *testing.T) { SnapshotFiles: []string{"fs/multitar.img", "fs/fscheckpoint.pb"}, DataSnapshotFiles: []string{"fs/fscheckpoint.pb", "fs/multitar.img"}, } - if err := RecordCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, volResp); err != nil { + if err := RecordCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, volResp); err != nil { t.Fatalf("RecordCheckpoint(VOLUMES): %v", err) } - got = ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES) + got = ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES) if got == nil || !slices.Equal(got.GetSnapshotFiles(), volResp.GetSnapshotFiles()) || !slices.Equal(got.GetDataSnapshotFiles(), volResp.GetDataSnapshotFiles()) { t.Fatalf("ReuseCheckpoint(VOLUMES) = %v, want %v", got, volResp) } // VOLUMES fidelity rejects non-data files in SnapshotFiles. - if err := RecordCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, memResp); err != nil { + if err := RecordCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, memResp); err != nil { t.Fatalf("RecordCheckpoint(VOLUMES with non-data files): %v", err) } - if got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES); got != nil { + if got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES); got != nil { t.Fatalf("ReuseCheckpoint(VOLUMES with non-data files) = %v, want nil", got) } } diff --git a/internal/bootid/bootid.go b/internal/bootid/bootid.go new file mode 100644 index 0000000000..07c630ea34 --- /dev/null +++ b/internal/bootid/bootid.go @@ -0,0 +1,41 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bootid reads the node's boot ID. +package bootid + +import ( + "fmt" + "os" + "strings" +) + +// Path is where Linux exposes the boot ID: a random UUID generated once per +// boot, so it changes on every reboot, clean or not, and on nothing else. It +// is not namespaced, so a pod reads the node's. +const Path = "/proc/sys/kernel/random/boot_id" + +// Read returns the boot ID read from path. It is never empty, so a record that +// has none never matches it. +func Read(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("while reading the boot ID: %w", err) + } + id := strings.TrimSpace(string(b)) + if id == "" { + return "", fmt.Errorf("boot ID file %s is empty", path) + } + return id, nil +} diff --git a/cmd/atelet/bootid_linux_test.go b/internal/bootid/bootid_linux_test.go similarity index 70% rename from cmd/atelet/bootid_linux_test.go rename to internal/bootid/bootid_linux_test.go index 0283181a3c..237fcff80b 100644 --- a/cmd/atelet/bootid_linux_test.go +++ b/internal/bootid/bootid_linux_test.go @@ -14,15 +14,15 @@ //go:build linux -package main +package bootid import "testing" -// TestReadBootIDFromHost reads the boot ID atelet reads at startup, which -// only Linux exposes. -func TestReadBootIDFromHost(t *testing.T) { - got, err := readBootID(bootIDPath) +// TestReadFromHost reads the boot ID atelet and the workers read at startup, +// which only Linux exposes. +func TestReadFromHost(t *testing.T) { + got, err := Read(Path) if err != nil || got == "" { - t.Errorf("readBootID(%q) = (%q, %v), want a non-empty boot ID", bootIDPath, got, err) + t.Errorf("Read(%q) = (%q, %v), want a non-empty boot ID", Path, got, err) } } diff --git a/internal/bootid/bootid_test.go b/internal/bootid/bootid_test.go new file mode 100644 index 0000000000..0dc8537891 --- /dev/null +++ b/internal/bootid/bootid_test.go @@ -0,0 +1,51 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bootid + +import ( + "os" + "path/filepath" + "testing" +) + +func TestRead(t *testing.T) { + t.Run("reads and trims boot ID", func(t *testing.T) { + p := filepath.Join(t.TempDir(), "boot_id") + const want = "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f" + if err := os.WriteFile(p, []byte(want+"\n"), 0o600); err != nil { + t.Fatal(err) + } + got, err := Read(p) + if err != nil || got != want { + t.Errorf("Read = (%q, %v), want (%q, nil)", got, err, want) + } + }) + + t.Run("rejects empty file", func(t *testing.T) { + p := filepath.Join(t.TempDir(), "boot_id") + if err := os.WriteFile(p, []byte(" \n\t"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Read(p); err == nil { + t.Error("Read accepted an empty boot ID file") + } + }) + + t.Run("rejects missing file", func(t *testing.T) { + if _, err := Read(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Error("Read accepted a missing file") + } + }) +} From 9a4cb2ca1a673fe28e704272904a8f92eeb8b138 Mon Sep 17 00:00:00 2001 From: Aditya Shantanu Date: Wed, 28 Oct 2026 23:54:14 +0000 Subject: [PATCH 44/53] atelet: keep the worker's Unavailable when a checkpoint fails (#3123) Partially addresses #3046. Draining a node under sync-suspend load crashed actors with `suspend failed: atelet Checkpoint: while calling worker CheckpointWorkload: rpc error: code = Unavailable ... dial unix /var/lib/ate/worker/...`. Checkpoint wrapped the worker's error with fmt.Errorf, so atelet's interceptor returned Internal and ateapi crashed the actor. Run already keeps a worker's Unavailable through `workerCallError`; Checkpoint now does the same, so ateapi returns Unavailable and keeps the actor SUSPENDING with its assignment for a retry. A worker whose pod is gone, or that restarted and registered again, still crashes its actors through the worker workflows. Unlike Restore there is no retry limit: a crash after a failed checkpoint loses the running actor's state. A worker that restarts and never registers (#3118) therefore leaves the actor SUSPENDING, with ResumeActor refused, until that issue is fixed; before this change the next suspend crashed it. Tests: `TestCheckpointKeepsWorkerUnavailable` (fails without the change), Canceled and DeadlineExceeded rows in `TestWorkerCallError`. `go test -race -count=3 -shuffle=on ./cmd/atelet/...`, gofmt and go vet are clean. AI use: code and tests written with Claude Code; a second Claude session reviewed the diff adversarially and its findings were applied. I reviewed the final diff and the test results by hand. --------- Signed-off-by: Aditya Shantanu Co-authored-by: Aditya Shantanu --- cmd/atelet/lifecycle_test.go | 30 ++++++++++++++++++++++++++++++ cmd/atelet/main.go | 15 +++++++++------ cmd/atelet/main_test.go | 4 +++- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 72eccf43d5..e51bdf6444 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -86,6 +86,9 @@ type fakeWorker struct { // restoreErr, if set, fails every RestoreWorkload after it recorded the // request, as a worker that could not start the sandbox would. restoreErr error + // checkpointErr, if set, fails every CheckpointWorkload after it recorded + // the request, before it writes any snapshot file. + checkpointErr error // actorDirs records the ActorDirs each RPC arrived with, by RPC name. actorDirs map[string]*ateworkerpb.ActorDirs // preserveRestoreDir records the PreserveRestoreDir flag from the most @@ -160,6 +163,9 @@ func (f *fakeWorker) RunWorkload(_ context.Context, req *ateworkerpb.RunWorkload func (f *fakeWorker) CheckpointWorkload(ctx context.Context, req *ateworkerpb.CheckpointWorkloadRequest) (*ateworkerpb.CheckpointWorkloadResponse, error) { f.recordActorDirs("CheckpointWorkload", req.GetActorDirs()) f.recordFidelity("CheckpointWorkload", req.GetFidelity()) + if f.checkpointErr != nil { + return nil, f.checkpointErr + } dir := req.GetActorDirs().GetCheckpointDir() if resp := f.stoppedBy(); resp != nil { for _, name := range resp.GetSnapshotFiles() { @@ -3961,3 +3967,27 @@ func TestCheckpointFinishesWhenCallerCancels(t *testing.T) { unlock() }) } + +// A worker that is unreachable when Checkpoint calls it fails the checkpoint +// with Unavailable, however many times in a row, so ateapi keeps the actor +// SUSPENDING for a retry instead of crashing an actor whose state only the +// worker holds. The retry checkpoints once the worker answers again. +func TestCheckpointKeepsWorkerUnavailable(t *testing.T) { + a := runPluginActor(t) + snap := a.snapshot("pause-snap-1", 1) + checkpointCode := func() codes.Code { + _, err := a.checkpoint(t, snap, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + return apierror.Code(err) + } + + a.worker.checkpointErr = status.Error(codes.Unavailable, `connection error: desc = "transport: Error while dialing: dial unix /var/lib/ate/worker/sock: connect: no such file or directory"`) + for i := 1; i <= 5; i++ { + if got := checkpointCode(); got != codes.Unavailable { + t.Fatalf("Checkpoint %d with the worker unreachable: code = %v, want %v", i, got, codes.Unavailable) + } + } + a.worker.checkpointErr = nil + if got := checkpointCode(); got != codes.OK { + t.Fatalf("Checkpoint with the worker back: code = %v, want %v", got, codes.OK) + } +} diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index a16b722839..640d79d88d 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -1111,9 +1111,9 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest }) dWorker = time.Since(tWorker) if err != nil { - // TODO: Worker should classify checkpoint failures, and set "should-crash" - // in the metadata if the error is not retriable. - return nil, fmt.Errorf("while calling worker CheckpointWorkload: %w", err) + // TODO(#3046): Worker should classify checkpoint failures that left + // the workload intact as retriable. + return nil, workerCallError("CheckpointWorkload", err) } // The snapshot commits on this node once worker has written it, in either // upload mode. @@ -1884,9 +1884,12 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res } // workerCallError is atelet's error for a failed call to the worker's rpc. A -// worker that is unavailable, refusing workloads until it registers or gone -// with a restart, is reported as Unavailable, so the caller retries once the -// worker process has registered. Anything else reaches the caller as Internal. +// worker that is unavailable, refusing workloads until it registers or +// unreachable because its process is gone, is reported as Unavailable, so +// ateapi keeps the actor's state and assignment for a retry instead of +// crashing it. Once a restarted worker registers again, or when its pod is +// gone, ateapi crashes the actors it hosted on its own. Anything else reaches +// the caller as Internal. func workerCallError(rpc string, err error) error { if status.Code(err) == codes.Unavailable { return apierror.Unavailable("worker %s: the worker is unavailable: %s", rpc, status.Convert(err).Message()) diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 4da514ea30..d3daf4e277 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -1657,7 +1657,7 @@ func TestServingCheck(t *testing.T) { // A worker that is unavailable, whether refusing workloads until it registers // or unreachable after a restart, reaches atelet's caller as Unavailable so the -// resume is retried. Any other worker failure is Internal. +// resume or suspend is retried. Any other worker failure is Internal. func TestWorkerCallError(t *testing.T) { for _, tc := range []struct { name string @@ -1667,6 +1667,8 @@ func TestWorkerCallError(t *testing.T) { {name: "worker refuses", err: status.Error(codes.Unavailable, "worker has not registered yet"), want: codes.Unavailable}, {name: "worker unreachable", err: fmt.Errorf("dial: %w", status.Error(codes.Unavailable, "connection refused")), want: codes.Unavailable}, {name: "worker fails", err: status.Error(codes.Internal, "sandbox would not start"), want: codes.Internal}, + {name: "worker canceled", err: status.Error(codes.Canceled, "gave up waiting for the actor's lock: context canceled"), want: codes.Internal}, + {name: "worker deadline exceeded", err: status.Error(codes.DeadlineExceeded, "gave up waiting for the actor's lock: context deadline exceeded"), want: codes.Internal}, {name: "plain error", err: errors.New("boom"), want: codes.Internal}, } { t.Run(tc.name, func(t *testing.T) { From ea4018ff0bbde70decc289f2956288d8e3075d00 Mon Sep 17 00:00:00 2001 From: igooch Date: Thu, 29 Oct 2026 00:08:17 +0000 Subject: [PATCH 45/53] atelet: clear node-local caches after a node reboot (#3376) Part of #3351. This is a workaround. The real fix is for the caches to flush their writes to disk. atelet doesn't sync cache writes, so an unclean reboot can leave cache entries, image layers and bundle overlay specs empty or truncated. atelet then serves them, every restore on the node fails, and the image cache GC stops on the unreadable spec. At startup, atelet now compares the node's boot ID with the one it recorded when the caches were last written (`/var/lib/ate/cache-boot-id`). If the node rebooted since, it empties the image, sandbox asset and snapshot caches, and removes each actor's `bundles` dir and each worker's `default-runtime` dir. Those hold specs naming cached layers and hard links to cached assets, which atelet would otherwise reuse. Run and the worker sandbox syncer recreate them. This is the approach #3050 took for local snapshots. The rest of an actor's dir stays. Run and Restore rebuild it before use, and Terminate of an actor that crashed in the reboot names no worker, so it doesn't read the actor's sandbox record or asset links. The cost is cold caches after any reboot. When the record is missing, atelet only records the boot, since actors may be running on the caches the first time it starts with this check. So a node that is already broken stays broken after the upgrade until it reboots again. Reboot or replace such a node once after upgrading. On a GKE node, I filled the snapshot and image caches, waited 10s and hard-reset the VM. On main this tore 92 image cache and 10 snapshot cache files, and both a golden restore and a new actor from the torn image failed with `unexpected end of JSON input`. With this change, the caches were cleared at startup and both resumes succeeded. I used an AI agent (Claude Code) to write this change and its test, test it on a GKE node with a hard reset, and draft this description. --------- Signed-off-by: Ivy --- cmd/atelet/bootid.go | 81 ++++++++++++++++++ cmd/atelet/bootid_test.go | 86 ++++++++++++++++++++ cmd/atelet/internal/ateletpath/ateletpath.go | 5 ++ cmd/atelet/main.go | 31 ++++--- docs/sandbox-asset-cache.md | 4 + internal/imagecache/README.md | 7 +- 6 files changed, 201 insertions(+), 13 deletions(-) diff --git a/cmd/atelet/bootid.go b/cmd/atelet/bootid.go index 32c9e6485d..715a2a5ecf 100644 --- a/cmd/atelet/bootid.go +++ b/cmd/atelet/bootid.go @@ -15,7 +15,16 @@ package main import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/cmd/atelet/internal/atomicfile" + "go.ate.dev/substrate/internal/imagecache" ) // localSnapshotBootCheck returns the snapshot tracker's check of whether an @@ -32,3 +41,75 @@ func localSnapshotBootCheck(bootID string) func(actorUID, snapshotUUID string) b return err == nil && rec.BootID == bootID } } + +// clearCachesAfterReboot empties each of cacheDirs, and removes every actor's +// OCI bundle dir and every worker pod's default-runtime dir, when markerPath +// records a boot other than bootID. Then it records bootID there. It reports +// whether it cleared anything. +// +// Cache writes aren't synced, so an unclean reboot can leave cache entries +// and bundle specs empty or truncated. The bundle and default-runtime dirs +// hold specs naming cached layers and hard links to cached assets, which +// atelet reuses instead of linking again. No sandbox survives a reboot, so +// nothing still uses any of them. Run recreates an actor's bundle dir, and the +// worker sandbox syncer recreates a worker's default-runtime dir. +// +// A missing marker only records bootID. atelet may be starting with this +// check for the first time while actors run on what it would clear. +func clearCachesAfterReboot(bootID, markerPath, actorsDir, workersDir string, cacheDirs ...string) (bool, error) { + b, err := os.ReadFile(markerPath) + switch { + case errors.Is(err, fs.ErrNotExist): + return false, writeCacheBootID(markerPath, bootID) + case err != nil: + return false, fmt.Errorf("while reading the cache boot ID: %w", err) + case strings.TrimSpace(string(b)) == bootID: + return false, nil + } + for _, dir := range cacheDirs { + if err := removeChildren(dir, ""); err != nil { + return false, err + } + } + if err := removeChildren(actorsDir, "bundles"); err != nil { + return false, err + } + if err := removeChildren(workersDir, defaultRuntimeDirsName); err != nil { + return false, err + } + return true, writeCacheBootID(markerPath, bootID) +} + +// removeChildren removes the named entry in each child directory of dir, or +// each child itself when name is empty. A missing dir has nothing to remove. +func removeChildren(dir, name string) error { + if dir == "" { + return nil + } + children, err := os.ReadDir(dir) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("while listing %s: %w", dir, err) + } + for _, child := range children { + if name != "" && !child.IsDir() { + continue + } + path := filepath.Join(dir, child.Name(), name) + // RemoveAllWritable, not os.RemoveAll: unpacked image layers keep + // the image's directory modes, which can be read-only. + if err := imagecache.RemoveAllWritable(path); err != nil { + return fmt.Errorf("while removing %s: %w", path, err) + } + } + return nil +} + +func writeCacheBootID(path, bootID string) error { + if err := atomicfile.WriteSync(path, []byte(bootID+"\n"), 0o600); err != nil { + return fmt.Errorf("while recording the cache boot ID: %w", err) + } + return nil +} diff --git a/cmd/atelet/bootid_test.go b/cmd/atelet/bootid_test.go index 89c6f3c4fd..2ef67806aa 100644 --- a/cmd/atelet/bootid_test.go +++ b/cmd/atelet/bootid_test.go @@ -15,6 +15,8 @@ package main import ( + "errors" + "io/fs" "os" "path/filepath" "testing" @@ -59,3 +61,87 @@ func TestLocalSnapshotBootCheck(t *testing.T) { }) } } + +// clearCachesAfterReboot clears the caches, the actors' bundle dirs and the +// workers' default-runtime dirs only once the node has rebooted since the +// recorded boot, and leaves everything else in place. +func TestClearCachesAfterReboot(t *testing.T) { + for _, tc := range []struct { + name string + // marker is the recorded boot ID, or empty for no marker file. + marker string + wantCleared bool + }{ + {name: "rebooted since the recorded boot", marker: "boot-1\n", wantCleared: true}, + {name: "same boot", marker: "boot-2\n"}, + {name: "no recorded boot", marker: ""}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + marker := filepath.Join(root, "cache-boot-id") + imageCache := filepath.Join(root, "image-cache") + assetCache := filepath.Join(root, "sandbox-asset-cache") + actors := filepath.Join(root, "actors") + workers := filepath.Join(root, "workers") + + cleared := []string{ + filepath.Join(imageCache, "layers", "sha256", "abc", "fs", "etc", "ca.crt"), + filepath.Join(imageCache, "version"), + filepath.Join(assetCache, "entries", "k1", "data"), + filepath.Join(actors, "a1", "bundles", "c1", "overlay.json"), + filepath.Join(workers, "w1", "default-runtime", "sha256-x", "runsc"), + } + kept := []string{ + filepath.Join(actors, "a1", "snapshot-state.json"), + filepath.Join(actors, "a1", "sandbox-assets.json"), + filepath.Join(workers, "w1", "sandbox-runtimes.json"), + filepath.Join(root, "snapshots", "a1", "local-checkpoint"), + } + for _, p := range append(append([]string{}, cleared...), kept...) { + if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, nil, 0o600); err != nil { + t.Fatal(err) + } + } + // A plain file directly in the actors dir has no bundle dir. + if err := os.WriteFile(filepath.Join(actors, "stray"), nil, 0o600); err != nil { + t.Fatal(err) + } + // Unpacked layers keep the image's directory modes. + readOnly := filepath.Join(imageCache, "layers", "sha256", "abc", "fs", "etc") + if err := os.Chmod(readOnly, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(readOnly, 0o700) }) + if tc.marker != "" { + if err := os.WriteFile(marker, []byte(tc.marker), 0o600); err != nil { + t.Fatal(err) + } + } + + got, err := clearCachesAfterReboot("boot-2", marker, actors, workers, imageCache, assetCache, "") + if err != nil { + t.Fatalf("clearCachesAfterReboot: %v", err) + } + if got != tc.wantCleared { + t.Errorf("clearCachesAfterReboot cleared = %v, want %v", got, tc.wantCleared) + } + for _, p := range cleared { + _, err := os.Stat(p) + if gone := errors.Is(err, fs.ErrNotExist); gone != tc.wantCleared { + t.Errorf("%s removed = %v, want %v (stat err %v)", p, gone, tc.wantCleared, err) + } + } + for _, p := range append(kept, imageCache, assetCache, filepath.Join(actors, "stray")) { + if _, err := os.Stat(p); err != nil { + t.Errorf("%s should stay: %v", p, err) + } + } + if b, err := os.ReadFile(marker); err != nil || string(b) != "boot-2\n" { + t.Errorf("marker = (%q, %v), want (%q, nil)", b, err, "boot-2\n") + } + }) + } +} diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index f28ff09c45..6f7ee9b015 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -57,6 +57,11 @@ var ( // again. SandboxRuntimeIndexFile = filepath.Join(nodepath.BasePath, "sandbox-runtime-index.json") + // CacheBootIDFile records the boot ID of the boot that last wrote atelet's + // caches. atelet clears them at startup when the node has rebooted since + // (see clearCachesAfterReboot in package main). + CacheBootIDFile = filepath.Join(nodepath.BasePath, "cache-boot-id") + // SnapshotCacheDir is the node-local cache of shared snapshot files, the // snapshots many actors are created from (see // cmd/atelet/internal/filecache). It lives under BasePath because cache diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 640d79d88d..90926c2e07 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -254,9 +254,30 @@ func main() { imageCredsKeychain = kc } + // Validated before the caches are cleared, so a directory the flags + // reject is never emptied. if err := validateImageCacheGCFlags(); err != nil { serverboot.Fatal(ctx, "Invalid image cache GC flags", err) } + if err := validateSandboxAssetCacheFlags(); err != nil { + serverboot.Fatal(ctx, "Invalid sandbox asset cache flags", err) + } + if err := validateSnapshotCacheFlags(); err != nil { + serverboot.Fatal(ctx, "Invalid snapshot cache flags", err) + } + bootID, err := bootid.Read(bootid.Path) + if err != nil { + serverboot.Fatal(ctx, "Failed to read the node's boot ID", err) + } + cleared, err := clearCachesAfterReboot(bootID, ateletpath.CacheBootIDFile, nodepath.ActorsDir, nodepath.WorkersDir(), + *imageCacheDir, *sandboxAssetCacheDir, *snapshotCacheDir) + if err != nil { + serverboot.Fatal(ctx, "Failed to clear the caches after a reboot", err) + } + if cleared { + slog.InfoContext(ctx, "Cleared the node-local caches after a node reboot") + } + imageCache, err := imagecache.New(*imageCacheDir, imagecache.WithKeychain(imageCredsKeychain), imagecache.WithLocalhostRegistryReplacement(*localhostRegistryReplacement), @@ -271,9 +292,6 @@ func main() { go newImageCacheGC(imageCache, *imageCacheDir).Run(ctx) } - if err := validateSandboxAssetCacheFlags(); err != nil { - serverboot.Fatal(ctx, "Invalid sandbox asset cache flags", err) - } sandboxAssetCache, err := openSandboxAssetCache(ctx, *sandboxAssetCacheDir, *sandboxAssetCacheMinAge, *sandboxAssetFetchTimeout, nodepath.ActorsDir, nodepath.WorkersDir()) if err != nil { serverboot.Fatal(ctx, "Failed to open sandbox asset cache", err) @@ -286,9 +304,6 @@ func main() { if err := filecachegc.Start(ctx, ateattr.FileCacheSandboxAsset, sandboxAssetCache, sandboxAssetCacheGCFlags, fileCacheInstruments); err != nil { serverboot.Fatal(ctx, "Failed to start the sandbox asset cache GC", err) } - if err := validateSnapshotCacheFlags(); err != nil { - serverboot.Fatal(ctx, "Invalid snapshot cache flags", err) - } snapshotCache, err := openSnapshotCache(ctx, *snapshotCacheDir, *snapshotCacheMinAge, *snapshotCacheFetchTimeout) if err != nil { serverboot.Fatal(ctx, "Failed to open snapshot cache", err) @@ -385,10 +400,6 @@ func main() { defer ateapiConn.Close() workerService := ateapi.NewWorkerServiceClient(ateapiConn) - bootID, err := bootid.Read(bootid.Path) - if err != nil { - serverboot.Fatal(ctx, "Failed to read the node's boot ID", err) - } // The snapshot tracker's workers get a context of their own, cancelled only // once the gRPC server has drained, so background snapshot work keeps // running while in-flight RPCs finish. diff --git a/docs/sandbox-asset-cache.md b/docs/sandbox-asset-cache.md index 1b42d9524a..fca15052bf 100644 --- a/docs/sandbox-asset-cache.md +++ b/docs/sandbox-asset-cache.md @@ -28,6 +28,7 @@ cannot cross mounts. | `workers//default-runtime//` | The worker's default runtimes, one directory per default its file advertises, by bundle hash, one per sandbox class, linked in from the cache by the prewarmer. These links are what keeps a default runtime cached while no actor runs it; the syncer removes a directory when the file stops naming its runtime, and they all go with the worker's directory when the pod does. | | `sandbox-asset-scratch/` | Short-lived destinations for fetches that have no actor and no worker directory to link into: the assets the worker sandbox syncer stages when checking a version (and probes `runsc` for CPU features). Wiped at startup. | | `sandbox-runtime-index.json` | The runtime index serialized: every runtime a worker on this node has advertised, by hash, with the asset record the hash was computed from. | +| `cache-boot-id` | The node's boot ID when `atelet` last started. A different ID at startup means the node rebooted, so `atelet` empties the cache first. | A cache entry is keyed by the asset's sha256, so the same binaries under another name or from another bucket are one entry. The store never @@ -129,6 +130,9 @@ which directories exist. Nothing counts references and nothing prunes. restart the directories of pods still on the node keep protecting their defaults with no wait on the syncer, and the startup sweep removes the directories of pods that are gone. +- Cache writes aren't synced, so after a node reboot atelet empties the cache + and removes every worker's `default-runtime/` directory at startup. The + syncer and the prewarmer then rebuild them as on a new node. The old default's directory is removed as soon as the file names the new default: by then the worker has registered the new default (and unless an diff --git a/internal/imagecache/README.md b/internal/imagecache/README.md index 7e3f2d198e..db37c4c2e3 100644 --- a/internal/imagecache/README.md +++ b/internal/imagecache/README.md @@ -24,9 +24,10 @@ What it buys, concretely: single-digit milliseconds on warm nodes. - Layers shared between images are **downloaded and unpacked once per node**, not once per image; actors sharing layers also share page cache. -- The cache is **on disk and survives atelet restarts and node reboots** - (the old in-memory cache was lost on every restart, and its unbounded heap - retention could OOM atelet — #437). +- The cache is **on disk and survives atelet restarts** (the old in-memory + cache was lost on every restart, and its unbounded heap retention could OOM + atelet — #437). Its writes aren't synced, so atelet empties it at startup + after a node reboot. - **Tag refs are cacheable**: a tag costs one `HEAD` request to resolve to a manifest digest (the only safe cache key for mutable tags); digest refs hit the cache with zero network I/O. From d86746d839db6142ec0aea62db9c4fd280fa18f9 Mon Sep 17 00:00:00 2001 From: Sneha-at Date: Thu, 29 Oct 2026 00:22:31 +0000 Subject: [PATCH 46/53] Reject snapshotConfig.storageLocation without its preview gate (#3357) Follow-up to #2544, from review comment https://github.com/agent-substrate/substrate/pull/2544#discussion_r4365502973. `snapshotConfig.storageLocation` was accepted and ignored while the `PerTemplateSnapshotLocation` gate was off. A user could name a bucket, get a success response, and have snapshots written under the cluster `--snapshot-location`. ### What changed? Tested on kind with the cluster `--snapshot-location=gs://ate-snapshots/`. `` is the template's `storageLocation`. | Gate | Case | Before (`7e6476cbd`) | This PR | |---|---|---|---| | ON | with `storageLocation`, `--preview` | created, snapshot under `` | created, snapshot under `` | | ON | with `storageLocation`, no `--preview` | rejected: `Forbidden: requires a client that opts in to preview features ...` | rejected: `Forbidden` | | ON | without `storageLocation` | created, snapshot under `gs://ate-snapshots/` | same | | OFF | with `storageLocation`, `--preview` | created, but snapshot under `gs://ate-snapshots/`, not `` | rejected: `Forbidden` | | OFF | with `storageLocation`, no `--preview` | created, field ignored | rejected: `Forbidden` | | OFF | without `storageLocation` | created, snapshot under `gs://ate-snapshots/` | same | ### API validation - With preview off, a template that sets `storageLocation` is rejected with `InvalidArgument`. Before, it was accepted and silently ignored. - The old custom check is replaced by the standard ifDisabled forbidden rule that other preview fields use. ### Demos and benchmarks - No demo or benchmark template sets `storageLocation` any more. They all store snapshots under the cluster `--snapshot-location`. - `${BUCKET_NAME}` templating is gone from the demo rendering and the benchmark deploy scripts. The traffic-driver script loses `--bucket-name`. ### Install (ate-setup) - The comments and operator docs now say that the demos, benchmarks and e2e need `SNAPSHOT_LOCATION` even with the gate on. ### E2E fixtures and suites - With the gate on, the fixtures, combinedvolumes, and the demo, golden and template-update tests each set their own per-template location through `e2e.SnapshotLocation`. With it off, they leave the field out. - Templates copied from a demo also get their own location with the gate on. This brings per-template coverage back to the networkpolicy and egress suites. ### Unit tests - Validation tests now expect `Forbidden` when preview is off. Length and URI checks run with preview on. - The control API tests no longer rely on a default `storageLocation`. They set the cluster default the way nearby tests do. - The demo manifest tests and the kubectl-ate template tests fail if a shipped manifest sets `storageLocation` again. ### Docs - The API guide, operator install docs, demo READMEs, benchmarking README and the preview gate doc no longer mention bucket templating. They describe the field as preview-only. --------- Signed-off-by: Sneha Aradhey --- api/ateapi/v1/ateapi.pb.go | 3 +- api/ateapi/v1/ateapi.proto | 3 +- benchmarking/README.md | 3 +- benchmarking/deploy_locust.sh | 2 +- .../nighthawk-egress/deploy-traffic-driver.sh | 16 +------- .../traffic-driver-template.yaml.tmpl | 3 +- benchmarking/workloads/deploy.sh | 9 +---- .../manifests/ate-env-template.yaml.tmpl | 1 - .../glutton-durdir-data-template.yaml.tmpl | 1 - .../glutton-durdir-full-template.yaml.tmpl | 1 - .../glutton-storage-template.yaml.tmpl | 1 - .../manifests/glutton-template.yaml.tmpl | 1 - .../manifests/kernelmem-template.yaml.tmpl | 1 - .../manifests/sleep-template.yaml.tmpl | 1 - .../sweperf-astropy-7336-template.yaml.tmpl | 3 +- .../sweperf-django-11099-template.yaml.tmpl | 3 +- ...weperf-matplotlib-20826-template.yaml.tmpl | 3 +- .../sweperf-sphinx-8638-template.yaml.tmpl | 3 +- .../sweperf-sympy-13647-template.yaml.tmpl | 3 +- .../sweperf/sweperf-template.yaml.j2 | 3 +- .../test_generate_sweperf_templates.py | 2 - .../manifests/usermem-template.yaml.tmpl | 1 - cmd/ate-setup/commands.md | 2 +- cmd/ate-setup/internal/config/config.go | 8 +++- cmd/ate-setup/internal/config/setting.go | 2 +- .../demos/claudemultiplex/claudemultiplex.go | 5 +-- .../internal/demos/demotest/demotest.go | 6 ++- cmd/ate-setup/internal/demos/render.go | 14 +------ cmd/ate-setup/internal/demos/substrate.go | 8 ++-- cmd/ate-setup/internal/steps/apienv.go | 4 +- .../internal/steps/substrate_test.go | 9 +++-- .../internal/apivalidation/actor_template.go | 16 +------- .../apivalidation/actor_template_test.go | 39 ++++++++++++------- .../apivalidation/zz_generated.validation.go | 7 ++++ .../controlapi/actor_template_test.go | 13 ++++--- .../functionaltest/actor_template_test.go | 29 ++++++-------- .../controlapi/functionaltest/actor_test.go | 16 +++----- .../controlapi/functionaltest/common_test.go | 10 +---- .../internal/cmd/actortemplate_test.go | 17 ++++---- demos/claude-code-multiplex/README.md | 5 ++- .../agent-luna-template.yaml.tmpl | 1 - .../agent-mars-template.yaml.tmpl | 1 - .../agent-orion-template.yaml.tmpl | 1 - demos/counter/README.md | 6 +-- .../counter-microvm-csi-test-template.yaml | 1 - .../counter-microvm-template.yaml.tmpl | 1 - demos/counter/counter-template.yaml.tmpl | 1 - .../egress/egress-microvm-template.yaml.tmpl | 1 - demos/egress/egress-template.yaml.tmpl | 1 - demos/jupyter/README.md | 1 + demos/jupyter/jupyter-template.yaml.tmpl | 1 - demos/multi-template/README.md | 6 +-- .../multi-template/counter-template.yaml.tmpl | 1 - .../fspersist-template.yaml.tmpl | 1 - demos/parking/README.md | 6 +-- demos/parking/parking-template.yaml.tmpl | 1 - demos/sandbox/README.md | 4 +- demos/sandbox/manual-test-multi-template.yaml | 1 - demos/sandbox/sandbox-template.yaml.tmpl | 1 - docs/api-guide.md | 12 +++--- docs/csi-volumes.md | 2 - docs/operator-install.md | 6 ++- internal/e2e/fixture.go | 18 ++++++--- .../capabilities-templates.yaml.tmpl | 6 --- .../probe/probe-bootfetch-template.yaml.tmpl | 2 - .../probe/probe-multiactor-template.yaml.tmpl | 2 - .../probe/probe-parking-template.yaml.tmpl | 2 - .../probe/probe-sized-template.yaml.tmpl | 2 - .../fixtures/probe/probe-template.yaml.tmpl | 2 - .../security/atespace-template.yaml.tmpl | 2 - .../testserver/grpcecho-template.yaml.tmpl | 2 - .../testserver/websocket-template.yaml.tmpl | 2 - internal/e2e/preview.go | 16 ++++++++ internal/e2e/preview_test.go | 21 ++++++++++ internal/e2e/probe.go | 4 +- internal/e2e/probe_test.go | 2 +- internal/e2e/sandbox.go | 9 ++--- internal/e2e/sandbox_test.go | 24 ++++++------ internal/e2e/suites/authz/authz_test.go | 5 +-- internal/e2e/suites/authz/helpers_test.go | 4 +- .../suites/capabilities/capabilities_test.go | 9 ++--- .../combinedvolumes/combinedvolumes_test.go | 9 +---- internal/e2e/suites/demo/demo_test.go | 26 +++++++------ internal/e2e/suites/demo/golden_test.go | 11 +----- .../e2e/suites/demo/updatetemplate_test.go | 32 +++++++-------- .../egresscredinject/egresscredinject_test.go | 5 +-- .../e2e/suites/egressmitm/egressmitm_test.go | 5 +-- .../e2e/suites/egressmitm/goldenboot_test.go | 5 +-- internal/e2e/suites/identity/identity_test.go | 5 +-- .../e2e/suites/multiactor/multiactor_test.go | 5 +-- .../e2e/suites/networking/grpcingress_test.go | 9 ++--- .../networking/websocketingress_test.go | 5 +-- internal/e2e/suites/parking/parking_test.go | 6 +-- .../e2e/suites/security/crossatespace_test.go | 11 +++--- internal/e2e/suites/sizing/sizing_test.go | 9 ++--- .../snapshotcredentials_test.go | 14 +++---- internal/e2e/template.go | 10 ++++- internal/preview/preview.go | 7 ++-- 98 files changed, 283 insertions(+), 359 deletions(-) diff --git a/api/ateapi/v1/ateapi.pb.go b/api/ateapi/v1/ateapi.pb.go index b61e08acf3..44de8a372c 100644 --- a/api/ateapi/v1/ateapi.pb.go +++ b/api/ateapi/v1/ateapi.pb.go @@ -3135,9 +3135,10 @@ type SnapshotConfig struct { // storage_location is the base object-storage URI snapshots of actors on // this version are stored under. If unset, ate-api-server's // --snapshot-location is used. Requires the PerTemplateSnapshotLocation preview - // gate; ignored without it. + // gate. // // +ate:preview=PerTemplateSnapshotLocation + // +k8s:ifDisabled(PerTemplateSnapshotLocation)=+k8s:forbidden // +k8s:optional // +k8s:maxLength=1024 // +k8s:customValidation # Validate URI diff --git a/api/ateapi/v1/ateapi.proto b/api/ateapi/v1/ateapi.proto index fb6c701357..7a59f6a098 100644 --- a/api/ateapi/v1/ateapi.proto +++ b/api/ateapi/v1/ateapi.proto @@ -1279,9 +1279,10 @@ message SnapshotConfig { // storage_location is the base object-storage URI snapshots of actors on // this version are stored under. If unset, ate-api-server's // --snapshot-location is used. Requires the PerTemplateSnapshotLocation preview - // gate; ignored without it. + // gate. // // +ate:preview=PerTemplateSnapshotLocation + // +k8s:ifDisabled(PerTemplateSnapshotLocation)=+k8s:forbidden // +k8s:optional // +k8s:maxLength=1024 // +k8s:customValidation # Validate URI diff --git a/benchmarking/README.md b/benchmarking/README.md index 9c17596d03..803989d54b 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -12,12 +12,11 @@ scenario ladder, read [observability.md](observability.md). > [!IMPORTANT] > Source the environment configuration file (e.g., `source .ate-dev-env.sh`) -> first so `PROJECT_ID`, `BUCKET_NAME`, etc. are set. +> first so `PROJECT_ID` etc. are set. > > **For Local Kind Clusters**: Set `KO_DOCKER_REPO` to the local registry so images are pulled without remote cloud credentials: > ```bash > export KO_DOCKER_REPO="localhost:5001" -> export BUCKET_NAME="ate-snapshots" > ``` > [!IMPORTANT] diff --git a/benchmarking/deploy_locust.sh b/benchmarking/deploy_locust.sh index 6d216125cd..d9421a6b7c 100755 --- a/benchmarking/deploy_locust.sh +++ b/benchmarking/deploy_locust.sh @@ -63,7 +63,7 @@ usage() { echo " -h|--help Show this help message" echo "" echo "Environment:" - echo " PROJECT_ID, BUCKET_NAME, etc. are read from .ate-dev-env.sh by the" + echo " PROJECT_ID etc. are read from .ate-dev-env.sh by the" echo " scripts this wrapper invokes." } diff --git a/benchmarking/nighthawk-egress/deploy-traffic-driver.sh b/benchmarking/nighthawk-egress/deploy-traffic-driver.sh index 825cf3dd2c..19d3aeb9ba 100755 --- a/benchmarking/nighthawk-egress/deploy-traffic-driver.sh +++ b/benchmarking/nighthawk-egress/deploy-traffic-driver.sh @@ -35,7 +35,6 @@ EGRESS_POLICY_MANIFEST="${SCRIPT_DIR}/egress-policy.yaml" KUBECTL_ATE_BIN="kubectl-ate" WAIT_TIMEOUT_SECS=300 CONTEXT="${KUBECTL_CONTEXT:-}" -BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" KO_DOCKER_REPO="${KO_DOCKER_REPO:-localhost:5001}" usage() { @@ -44,7 +43,6 @@ usage() { echo "Options:" echo " --deploy Deploy the ${ATESPACE}/${TEMPLATE_NAME} worker pool, actor template, and ${ACTOR_NAME} actor (default)" echo " --delete Delete the ${ACTOR_NAME} actor, template, atespace, and worker pool" - echo " --bucket-name BUCKET GCS bucket name for actor snapshots (default: \$BUCKET_NAME)" echo " --ko-docker-repo REPO Container registry for ko image builds (default: \$KO_DOCKER_REPO)" echo " --context CONTEXT kubectl context to use (default: \$KUBECTL_CONTEXT or current context)" echo " --wait-timeout SECONDS Timeout waiting for worker pool and golden snapshot readiness (default: ${WAIT_TIMEOUT_SECS})" @@ -60,13 +58,6 @@ while [[ "$#" -gt 0 ]]; do --delete) action="delete" ;; - --bucket-name) - shift - BUCKET_NAME="$1" - ;; - --bucket-name=*) - BUCKET_NAME="${1#*=}" - ;; --ko-docker-repo) shift KO_DOCKER_REPO="$1" @@ -150,10 +141,6 @@ wait_template_ready() { } deploy() { - if [[ -z "${BUCKET_NAME}" ]]; then - echo "Error: BUCKET_NAME is not set. Pass --bucket-name or configure .ate-dev-env.sh." >&2 - exit 1 - fi if [[ -z "${KO_DOCKER_REPO}" ]]; then echo "Error: KO_DOCKER_REPO is not set. Pass --ko-docker-repo or configure .ate-dev-env.sh." >&2 exit 1 @@ -181,8 +168,7 @@ deploy() { run_kubectl_ate delete actor-template "${TEMPLATE_NAME}" -a "${ATESPACE}" >/dev/null 2>&1 || true echo "Creating actor template ${ATESPACE}/${TEMPLATE_NAME} from ${MANIFEST}..." - sed "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" "${MANIFEST}" \ - | run_kubectl_ate create actor-template -f - + run_kubectl_ate create actor-template -f "${MANIFEST}" wait_template_ready diff --git a/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl b/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl index 27c68f984b..0960539262 100644 --- a/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl +++ b/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl @@ -14,7 +14,7 @@ # Substrate ActorTemplate resource (protojson-shaped ateapi.ActorTemplate, # NOT the ate.dev/v1 CRD) for the Nighthawk egress traffic-driver actor. -# Applied with `kubectl-ate create actor-template -f -` by +# Applied with `kubectl-ate create actor-template -f` by # deploy-traffic-driver.sh; the `nighthawk` atespace must exist. # # Runs Envoy's `nighthawk_service` gRPC server on port 80 (the default actor @@ -65,4 +65,3 @@ resources: memoryBytes: 1073741824 # 1Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/nighthawk/traffic-driver/ diff --git a/benchmarking/workloads/deploy.sh b/benchmarking/workloads/deploy.sh index 7881ca2820..b934326ead 100755 --- a/benchmarking/workloads/deploy.sh +++ b/benchmarking/workloads/deploy.sh @@ -23,12 +23,6 @@ if [[ -f .ate-dev-env.sh ]]; then source .ate-dev-env.sh fi -# Ensure BUCKET_NAME is set -if [[ -z "${BUCKET_NAME:-}" ]]; then - echo "Error: BUCKET_NAME environment variable is not set." >&2 - exit 1 -fi - MANIFEST_DIR="benchmarking/workloads/manifests" POOL_MANIFEST="${MANIFEST_DIR}/workloads.yaml.tmpl" # Generated sweperf ActorTemplates; see manifests/sweperf/generate_sweperf_templates.py. @@ -192,8 +186,7 @@ substitute() { if [[ -n "${WORKER_MEMORY}" ]]; then worker_resources="resources: {requests: {memory: \"${WORKER_MEMORY}\"}, limits: {memory: \"${WORKER_MEMORY}\"}}" fi - sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \ - -e "s|\${WORKER_COUNT}|${WORKER_COUNT}|g" \ + sed -e "s|\${WORKER_COUNT}|${WORKER_COUNT}|g" \ -e "s|\${SANDBOX_CLASS}|${SANDBOX_CLASS}|g" \ -e "s|\${OTLP_ENDPOINT}|${OTLP_ENDPOINT}|g" \ -e "s|\${ACTOR_MEMORY}|${ACTOR_MEMORY}|g" \ diff --git a/benchmarking/workloads/manifests/ate-env-template.yaml.tmpl b/benchmarking/workloads/manifests/ate-env-template.yaml.tmpl index 64d6791643..1df6a48b65 100644 --- a/benchmarking/workloads/manifests/ate-env-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/ate-env-template.yaml.tmpl @@ -44,4 +44,3 @@ resources: memoryBytes: "${ACTOR_MEMORY_BYTES}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/ate-env/ diff --git a/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl index d507e18483..67f510453b 100644 --- a/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl @@ -50,4 +50,3 @@ resources: memoryBytes: "${ACTOR_MEMORY_BYTES}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-data/ diff --git a/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl index 0cbc965059..f961cbdbe2 100644 --- a/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl @@ -47,4 +47,3 @@ resources: memoryBytes: "${ACTOR_MEMORY_BYTES}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-full/ diff --git a/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl index ad7dcb28bb..77fc654a40 100644 --- a/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl @@ -57,4 +57,3 @@ resources: memoryBytes: "${ACTOR_MEMORY_BYTES}" snapshotConfig: onCommit: SNAPSHOT_CONTENT_SCOPE_FULL - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-storage/ diff --git a/benchmarking/workloads/manifests/glutton-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-template.yaml.tmpl index 7ed6d25d1b..bdec96ac0f 100644 --- a/benchmarking/workloads/manifests/glutton-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-template.yaml.tmpl @@ -59,4 +59,3 @@ resources: memoryBytes: "${ACTOR_MEMORY_BYTES}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton/ diff --git a/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl b/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl index 7019662db9..df3f93c811 100644 --- a/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl @@ -37,4 +37,3 @@ resources: memoryBytes: 8589934592 # 8Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/kernelmem/ diff --git a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl index 57c8c5fbef..19c462199e 100644 --- a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl @@ -33,4 +33,3 @@ resources: memoryBytes: "${ACTOR_MEMORY_BYTES}" snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sleep/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl index 33bb0d7738..c641f74f27 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-astropy-7336-template.yaml.tmpl @@ -14,7 +14,7 @@ # Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py # from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this -# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +# file. metadata: atespace: benchmark-workloads name: sweperf-astropy-7336 @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-astropy-7336/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl index af94623150..4c5a62be26 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-django-11099-template.yaml.tmpl @@ -14,7 +14,7 @@ # Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py # from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this -# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +# file. metadata: atespace: benchmark-workloads name: sweperf-django-11099 @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-django-11099/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl index 040e0486ff..8751fbde91 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-matplotlib-20826-template.yaml.tmpl @@ -14,7 +14,7 @@ # Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py # from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this -# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +# file. metadata: atespace: benchmark-workloads name: sweperf-matplotlib-20826 @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-matplotlib-20826/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl index 1a8264c447..98c2222fd8 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sphinx-8638-template.yaml.tmpl @@ -14,7 +14,7 @@ # Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py # from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this -# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +# file. metadata: atespace: benchmark-workloads name: sweperf-sphinx-8638 @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-sphinx-8638/ diff --git a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl index e869b8c2ee..419de76702 100644 --- a/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sweperf/generated/sweperf-sympy-13647-template.yaml.tmpl @@ -14,7 +14,7 @@ # Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py # from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this -# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +# file. metadata: atespace: benchmark-workloads name: sweperf-sympy-13647 @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sweperf-sympy-13647/ diff --git a/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 b/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 index 9fc60b1653..a624026dae 100644 --- a/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 +++ b/benchmarking/workloads/manifests/sweperf/sweperf-template.yaml.j2 @@ -14,7 +14,7 @@ # Generated by benchmarking/workloads/manifests/sweperf/generate_sweperf_templates.py # from sweperf-images.json and sweperf-template.yaml.j2; edit those, not this -# file. The ${...} values are filled in by benchmarking/workloads/deploy.sh. +# file. metadata: atespace: benchmark-workloads name: {{ name }} @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/{{ name }}/ diff --git a/benchmarking/workloads/manifests/sweperf/test_generate_sweperf_templates.py b/benchmarking/workloads/manifests/sweperf/test_generate_sweperf_templates.py index 2d67908e01..e5c8242c67 100644 --- a/benchmarking/workloads/manifests/sweperf/test_generate_sweperf_templates.py +++ b/benchmarking/workloads/manifests/sweperf/test_generate_sweperf_templates.py @@ -40,8 +40,6 @@ def test_render_fills_task_values(self): self.assertIn("name: sweperf-django-11099", content) self.assertIn(f'image: "{PINNED}"', content) self.assertIn('value: "15"', content) - # deploy.sh fills these, so they must survive rendering. - self.assertIn("${BUCKET_NAME}", content) def test_check_reports_missing_stale_and_orphaned_files(self): rendered = gen.render([{"name": n, "image": PINNED, "steps": 1} diff --git a/benchmarking/workloads/manifests/usermem-template.yaml.tmpl b/benchmarking/workloads/manifests/usermem-template.yaml.tmpl index 5b0bd1fc54..84637bdcb4 100644 --- a/benchmarking/workloads/manifests/usermem-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/usermem-template.yaml.tmpl @@ -35,4 +35,3 @@ resources: memoryBytes: 1610612736 # 1536Mi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/usermem/ diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 28ed5d6fcf..87672d5217 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -192,7 +192,7 @@ is absent); the other two are flags of the benchmark commands. See | `deploy demo sandbox` | An on-demand sandbox actor driven by the sandbox client | | `deploy demo multi-template` | Two ActorTemplates sharing one WorkerPool | | `deploy demo parking` | Actor parking and unparking on a small WorkerPool | -| `deploy demo claude-code-multiplex` | Several Claude Code agents multiplexed onto one WorkerPool (requires `ANTHROPIC_API_KEY`, `BUCKET_NAME`, `KO_DOCKER_REPO`) | +| `deploy demo claude-code-multiplex` | Several Claude Code agents multiplexed onto one WorkerPool (requires `ANTHROPIC_API_KEY`, `KO_DOCKER_REPO`) | Each has a matching `delete demo NAME`. Demo flags bind to the deploy side only; teardown never reads them. diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index bf300d7469..2706b81cad 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -129,14 +129,18 @@ type Config struct { // defaults to the in-cluster URL of the install's ate-idp-server Service. ActorJWTIssuer string - // BucketName is the snapshot bucket demos are templated with. + // BucketName is the object-storage bucket (BUCKET_NAME) passed to install + // scripts such as hack/install-microvm-deps.sh. The kind profile also + // derives SnapshotLocation from it. BucketName string // SnapshotLocation is ate-api-server's --snapshot-location // (SNAPSHOT_LOCATION): the cluster-wide object-storage URI actor snapshots // are stored under. Required unless the PerTemplateSnapshotLocation // preview gate is enabled, which also lets a - // template's snapshotConfig.storageLocation override it. + // template's snapshotConfig.storageLocation override it. The demos and + // benchmarks, and so e2e runs, need it even with the gate, since they do + // not set storageLocation. SnapshotLocation string // TransferGSAName is the name of the Google service account snapshot diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 1fdea0effb..0c16e1b897 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -147,7 +147,7 @@ var Registry = []Setting{ { Key: "storage.bucketName", Env: "BUCKET_NAME", Flag: "storage-bucket-name", Kind: KindString, - Usage: "Snapshot bucket the demos are templated with", + Usage: "Object-storage bucket passed to install scripts; the kind profile derives --snapshot-location from it", }, { Key: "storage.snapshotLocation", Env: "SNAPSHOT_LOCATION", Flag: "snapshot-location", Kind: KindString, diff --git a/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex.go b/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex.go index f8b7db51b3..b89587f23e 100644 --- a/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex.go +++ b/cmd/ate-setup/internal/demos/claudemultiplex/claudemultiplex.go @@ -53,7 +53,7 @@ func init() { d := &demo{} d.sub = demos.Substrate{ DemoName: "demo-claude-code-multiplex", - Short: "Several Claude Code agents multiplexed onto one WorkerPool (requires ANTHROPIC_API_KEY, BUCKET_NAME, KO_DOCKER_REPO)", + Short: "Several Claude Code agents multiplexed onto one WorkerPool (requires ANTHROPIC_API_KEY, KO_DOCKER_REPO)", WorkerPoolManifest: poolManifest, Deployments: []steps.TemplateRef{{Atespace: atespace, Name: "claude-workerpool"}}, Templates: agentTemplates(), @@ -85,9 +85,6 @@ func (d *demo) Deploy(ctx context.Context, e *steps.Env) error { if e.Cfg.AnthropicAPIKey == "" { return fmt.Errorf("ANTHROPIC_API_KEY must be set") } - if e.Cfg.BucketName == "" { - return fmt.Errorf("BUCKET_NAME must be set") - } if e.Cfg.KODockerRepo == "" { return fmt.Errorf("KO_DOCKER_REPO must be set (see hack/ate-dev-env.sh.example)") } diff --git a/cmd/ate-setup/internal/demos/demotest/demotest.go b/cmd/ate-setup/internal/demos/demotest/demotest.go index 5e845ba967..aeef03a372 100644 --- a/cmd/ate-setup/internal/demos/demotest/demotest.go +++ b/cmd/ate-setup/internal/demos/demotest/demotest.go @@ -48,7 +48,7 @@ func Env(t *testing.T) *steps.Env { if err != nil { t.Fatalf("Resolve: %v", err) } - cfg := &config.Config{Root: root, BucketName: "ate-snapshots"} + cfg := &config.Config{Root: root} cfg.SetResolved(r) return &steps.Env{Cfg: cfg} } @@ -85,6 +85,10 @@ func AssertRenderedActorTemplate(t *testing.T, manifest []byte, want resources.A if got := resources.ActorTemplateRefFromActorTemplate(template); got != want { t.Errorf("manifest names template %s, want %s", got, want) } + // The API rejects storageLocation without its preview gate. + if got := template.GetSnapshotConfig().GetStorageLocation(); got != "" { + t.Errorf("manifest sets storageLocation %q, want none", got) + } } // assertNoPlaceholders checks that no non-comment line still contains a diff --git a/cmd/ate-setup/internal/demos/render.go b/cmd/ate-setup/internal/demos/render.go index a755057dfe..2d12137879 100644 --- a/cmd/ate-setup/internal/demos/render.go +++ b/cmd/ate-setup/internal/demos/render.go @@ -19,10 +19,6 @@ import ( "go.ate.dev/substrate/cmd/ate-setup/internal/steps" ) -// bucketNamePlaceholder is substituted into every demo template with the -// snapshot bucket for this environment. -const bucketNamePlaceholder = "BUCKET_NAME" - // ExternalVolumePlaceholders are the optional external-volume hooks in the // counter templates. Every path except // `deploy demo counter --with-external-volume` drops them, which removes the @@ -33,13 +29,7 @@ var ExternalVolumePlaceholders = []string{ "EXTERNAL_VOLUMES", } -// Render expands a demo template with the configured bucket name. -func Render(e *steps.Env, relPath string, extraValues map[string]string, drop []string) ([]byte, error) { - values := map[string]string{ - bucketNamePlaceholder: e.Cfg.BucketName, - } - for k, v := range extraValues { - values[k] = v - } +// Render expands a demo template with values. +func Render(e *steps.Env, relPath string, values map[string]string, drop []string) ([]byte, error) { return render.Template(e.Cfg.Path(relPath), values, drop) } diff --git a/cmd/ate-setup/internal/demos/substrate.go b/cmd/ate-setup/internal/demos/substrate.go index e5812d847b..6278efa2b2 100644 --- a/cmd/ate-setup/internal/demos/substrate.go +++ b/cmd/ate-setup/internal/demos/substrate.go @@ -68,10 +68,10 @@ type Substrate struct { // is a cloud-hypervisor cold boot plus a checkpoint, on nested KVM in CI. GoldenTimeout time.Duration - // RenderValues optionally supplies extra placeholder values at deploy - // time, for demos whose manifests need more than ${BUCKET_NAME} (e.g. a - // freshly built image digest). Delete renders only the pool manifest, so - // deploy-only values may live solely here. + // RenderValues optionally supplies placeholder values at deploy time, for + // demos whose manifests need them (e.g. a freshly built image digest). + // Delete renders only the pool manifest, so deploy-only values may live + // solely here. RenderValues func(ctx context.Context, e *steps.Env) (map[string]string, error) } diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 1fc1e4e3d0..755b346af0 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -187,7 +187,9 @@ func sharedBootstrapOwnerWarning(bootstrapOwners, namespace string) string { // snapshots. It is used whether or not the PerTemplateSnapshotLocation preview // gate is enabled; with the gate, an actor template's // snapshotConfig.storageLocation overrides it. It may be empty only when the -// gate is enabled, in which case every template must set its own. +// gate is enabled. CreateActorTemplate then fails with FailedPrecondition for +// a template without storageLocation, which includes every demo and +// benchmark. func clusterSnapshotLocation(location string, gates []string) (string, error) { if location != "" { if err := resources.ValidateSnapshotLocation(location); err != nil { diff --git a/cmd/ate-setup/internal/steps/substrate_test.go b/cmd/ate-setup/internal/steps/substrate_test.go index d29c0ec174..0f748c8daf 100644 --- a/cmd/ate-setup/internal/steps/substrate_test.go +++ b/cmd/ate-setup/internal/steps/substrate_test.go @@ -40,8 +40,7 @@ func TestActorTemplateFromManifestDemos(t *testing.T) { t.Fatalf("RepoRoot: %v", err) } - // The values benchmarking/workloads/deploy.sh substitutes, beyond the - // ${BUCKET_NAME} every installer provides. + // The values benchmarking/workloads/deploy.sh substitutes. benchmarkValues := map[string]string{ "${ACTOR_CPU_MILLICORES}": "1000", "${ACTOR_MEMORY_BYTES}": "268435456", @@ -113,7 +112,7 @@ func TestActorTemplateFromManifestDemos(t *testing.T) { if err != nil { t.Fatalf("ReadFile: %v", err) } - manifest := strings.ReplaceAll(string(data), "${BUCKET_NAME}", "ate-snapshots") + manifest := string(data) for placeholder, value := range tc.values { manifest = strings.ReplaceAll(manifest, placeholder, value) } @@ -127,6 +126,10 @@ func TestActorTemplateFromManifestDemos(t *testing.T) { if got := resources.ActorTemplateRefFromActorTemplate(template); got != tc.want { t.Errorf("manifest names template %s, want %s", got, tc.want) } + // The API rejects storageLocation without its preview gate. + if got := template.GetSnapshotConfig().GetStorageLocation(); got != "" { + t.Errorf("manifest sets storageLocation %q, want none", got) + } }) } } diff --git a/cmd/ateapi/internal/apivalidation/actor_template.go b/cmd/ateapi/internal/apivalidation/actor_template.go index cad1b5aa34..492af02aae 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template.go +++ b/cmd/ateapi/internal/apivalidation/actor_template.go @@ -16,10 +16,8 @@ package apivalidation import ( "context" - "fmt" "go.ate.dev/substrate/api/ateapi/v1" - "go.ate.dev/substrate/internal/preview" "go.ate.dev/substrate/internal/resources" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/util/sets" @@ -166,19 +164,9 @@ func ValidateCustom_Container_Image(_ context.Context, _ operation.Operation, fl } // ValidateCustom_SnapshotConfig_StorageLocation ensures an -// ActorTemplate's snapshotConfig.location is a well-formed +// ActorTemplate's snapshotConfig.storageLocation is a well-formed // URI with a bucket, so a bad location fails fast. -// -// The field is ignored while the PerTemplateSnapshotLocation gate is off, so a -// declarative ifDisabled=forbidden would reject the templates that set it -// on servers without the gate. Instead, with the gate on, a request that -// did not opt in to preview features is forbidden from setting it. -func ValidateCustom_SnapshotConfig_StorageLocation(ctx context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { - if preview.IsEnabled(preview.GatePerTemplateSnapshotLocation) && preview.IsOptedOut(ctx) { - return field.ErrorList{field.Forbidden(fldPath, fmt.Sprintf( - "requires a client that opts in to preview features (kubectl ate --preview, or the %s header) while the %s preview gate is enabled", - preview.Header, preview.GatePerTemplateSnapshotLocation))} - } +func ValidateCustom_SnapshotConfig_StorageLocation(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { if err := resources.ValidateSnapshotLocation(*value); err != nil { return field.ErrorList{field.Invalid(fldPath, *value, err.Error())} } diff --git a/cmd/ateapi/internal/apivalidation/actor_template_test.go b/cmd/ateapi/internal/apivalidation/actor_template_test.go index a2d8de5e56..4c111dc51b 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_template_test.go @@ -167,9 +167,9 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) { })}, nil, }, { - "snapshot_config.storage_location is not needed when the preview gate is off", + "valid storage_location", &ateapi.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapi.ActorTemplate) { - tmpl.SnapshotConfig.StorageLocation = "" + tmpl.SnapshotConfig.StorageLocation = "gs://my-bucket/snapshots" })}, nil, }, { @@ -230,6 +230,10 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) { want = field.ErrorList{field.Forbidden(field.NewPath("actor_template", "golden_egress_policy"), "")} wantWithPreview = map[string]field.ErrorList{"GoldenEgressPolicy": tt.want, "ALL": tt.want} } + if tt.req.GetActorTemplate().GetSnapshotConfig().GetStorageLocation() != "" { + want = field.ErrorList{field.Forbidden(field.NewPath("actor_template", "snapshot_config", "storage_location"), "")} + wantWithPreview = map[string]field.ErrorList{"PerTemplateSnapshotLocation": tt.want, "ALL": tt.want} + } assertValidate(t, func() field.ErrorList { return ValidateCreateActorTemplateRequest(context.Background(), tt.req) }, want, wantWithPreview) @@ -241,6 +245,7 @@ func TestValidateCreateActorTemplateRequest_PreviewRestricted(t *testing.T) { setPreviewForTest(t, "ALL") req := &ateapi.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapi.ActorTemplate) { tmpl.Volumes = []*ateapi.Volume{{Name: "data", External: &ateapi.ExternalVolumeSource{Template: &ateapi.ExternalVolumeTemplate{CapacityBytes: 10 << 30, StorageClassName: "fast-ssd"}}}} + tmpl.SnapshotConfig.StorageLocation = "gs://my-bucket/snapshots" })} t.Run("allowed", func(t *testing.T) { @@ -256,9 +261,9 @@ func TestValidateCreateActorTemplateRequest_PreviewRestricted(t *testing.T) { }) } -func TestValidateCreateActorTemplateRequest_PerTemplateSnapshotLocationOptIn(t *testing.T) { +func TestValidateCreateActorTemplateRequest_PerTemplateSnapshotLocation(t *testing.T) { storageLocation := field.NewPath("actor_template", "snapshot_config", "storage_location") - withoutLocation := func(tmpl *ateapi.ActorTemplate) { tmpl.SnapshotConfig.StorageLocation = "" } + withLocation := func(tmpl *ateapi.ActorTemplate) { tmpl.SnapshotConfig.StorageLocation = "gs://my-bucket/snapshots" } tests := []struct { name string gates string @@ -266,29 +271,34 @@ func TestValidateCreateActorTemplateRequest_PerTemplateSnapshotLocationOptIn(t * mutate func(*ateapi.ActorTemplate) want field.ErrorList }{{ + name: "gate off, opted in, location set", + mutate: withLocation, + want: field.ErrorList{field.Forbidden(storageLocation, "")}, + }, { name: "gate off, opted out, location set", optOut: true, + mutate: withLocation, + want: field.ErrorList{field.Forbidden(storageLocation, "")}, }, { name: "gate off, opted out, no location", optOut: true, - mutate: withoutLocation, }, { name: "gate on, opted out, location set", gates: "PerTemplateSnapshotLocation", optOut: true, + mutate: withLocation, want: field.ErrorList{field.Forbidden(storageLocation, "")}, }, { name: "gate on, opted out, no location", gates: "PerTemplateSnapshotLocation", optOut: true, - mutate: withoutLocation, }, { - name: "gate on, opted in, location set", - gates: "PerTemplateSnapshotLocation", - }, { - name: "gate on, opted in, no location", + name: "gate on, opted in, location set", gates: "PerTemplateSnapshotLocation", - mutate: withoutLocation, + mutate: withLocation, + }, { + name: "gate on, opted in, no location", + gates: "PerTemplateSnapshotLocation", }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -471,13 +481,15 @@ func TestValidateActorTemplate(t *testing.T) { mutate: func(tmpl *ateapi.ActorTemplate) { tmpl.SnapshotConfig.StorageLocation = "gs://" + strings.Repeat("x", 1020) }, - want: field.ErrorList{field.TooLong(field.NewPath("snapshot_config", "storage_location"), nil, 1024).WithOrigin("maxLength")}, + want: field.ErrorList{field.Forbidden(field.NewPath("snapshot_config", "storage_location"), "")}, + wantWithPreview: map[string]field.ErrorList{"PerTemplateSnapshotLocation": {field.TooLong(field.NewPath("snapshot_config", "storage_location"), nil, 1024).WithOrigin("maxLength")}}, }, { name: "missing storage_location", mutate: func(tmpl *ateapi.ActorTemplate) { tmpl.SnapshotConfig.StorageLocation = "" }, // Optional with or without the gate: an empty field falls back to // --snapshot-location. - want: nil, + want: nil, + wantWithPreview: map[string]field.ErrorList{"PerTemplateSnapshotLocation": nil}, }, { name: "unspecified snapshot scope", mutate: func(tmpl *ateapi.ActorTemplate) { @@ -962,7 +974,6 @@ func validActorTemplate(mutations ...func(*ateapi.ActorTemplate)) *ateapi.ActorT Metadata: &ateapi.ResourceMetadata{Atespace: "ns1", Name: "tmpl-a"}, Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: "gs://my-bucket/snapshots", PreferredFidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index ab12828f3f..bfe1672f01 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -8285,6 +8285,13 @@ func Validate_SnapshotConfig( } // call field-attached validations earlyReturn := false + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "PerTemplateSnapshotLocation", false, validate.ForbiddenValue).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "PerTemplateSnapshotLocation", false, validate.OptionalValue).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index 5ff334cbd9..d346b43146 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -42,7 +42,6 @@ func validActorTemplate(mutations ...func(*ateapi.ActorTemplate)) *ateapi.ActorT Metadata: &ateapi.ResourceMetadata{Atespace: "ns1", Name: "tmpl-a"}, Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: "gs://my-bucket/snapshots", PreferredFidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, @@ -57,7 +56,9 @@ func validActorTemplate(mutations ...func(*ateapi.ActorTemplate)) *ateapi.ActorT // while the atespace is missing, and succeeds once the atespace exists. func TestCreateActorTemplate(t *testing.T) { persistence := newTestPersistence(t) - s := &RPCService{impl: newServiceImpl(persistence, nil)} + impl := newServiceImpl(persistence, nil) + impl.clusterSnapshotLocation = "gs://cluster-bucket/" + s := &RPCService{impl: impl} ctx := context.Background() req := func(atespace, name string) *ateapi.CreateActorTemplateRequest { return &ateapi.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapi.ActorTemplate) { @@ -83,7 +84,7 @@ func TestCreateActorTemplate(t *testing.T) { // TestCreateActorTemplate_RequiresSnapshotLocation pins that create rejects, // with FailedPrecondition, a template whose actors would have no snapshot -// location: neither an honored storageLocation nor --snapshot-location. +// location: neither storageLocation nor --snapshot-location. func TestCreateActorTemplate_RequiresSnapshotLocation(t *testing.T) { tests := []struct { name string @@ -95,11 +96,9 @@ func TestCreateActorTemplate_RequiresSnapshotLocation(t *testing.T) { }{{ name: "gate off, cluster location set", clusterSnapshotLocation: "gs://cluster-bucket/", - location: "gs://my-bucket/snapshots", wantCode: codes.OK, }, { name: "gate off, no cluster location", - location: "gs://my-bucket/snapshots", wantCode: codes.FailedPrecondition, }, { name: "gate on, location set, no cluster location", @@ -153,7 +152,9 @@ func TestCreateActorTemplate_RequiresSnapshotLocation(t *testing.T) { // the only guard. func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) { persistence := newTestPersistence(t) - s := &RPCService{impl: newServiceImpl(persistence, nil)} + impl := newServiceImpl(persistence, nil) + impl.clusterSnapshotLocation = "gs://cluster-bucket/" + s := &RPCService{impl: impl} ctx := context.Background() if _, err := persistence.CreateAtespace(ctx, &ateapi.Atespace{Metadata: &ateapi.ResourceMetadata{Name: "ns1"}}); err != nil { diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go index e589656a95..c8e6edaff2 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go @@ -42,10 +42,9 @@ func TestActorTemplateCRUD(t *testing.T) { created, err := tc.client.CreateActorTemplate(ctx, &ateapi.CreateActorTemplateRequest{ ActorTemplate: &ateapi.ActorTemplate{ - Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a"}, - Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, - SnapshotConfig: &ateapi.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 1<<30)}, + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a"}, + Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, + Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 1<<30)}, // Server-owned status on the request is ignored. Status: &ateapi.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapi.GoldenSnapshotStatus{ @@ -61,7 +60,6 @@ func TestActorTemplateCRUD(t *testing.T) { Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a", Version: 1}, Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: "gs://my-bucket/snapshots", PreferredFidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, GoldenSnapshotConfig: &ateapi.GoldenSnapshotConfig{Mode: ateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_ENABLED}, }, @@ -74,10 +72,9 @@ func TestActorTemplateCRUD(t *testing.T) { _, err = tc.client.CreateActorTemplate(ctx, &ateapi.CreateActorTemplateRequest{ ActorTemplate: &ateapi.ActorTemplate{ - Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a"}, - Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, - SnapshotConfig: &ateapi.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a"}, + Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, + Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, }, }) assertGrpcError(t, err, codes.AlreadyExists, "ActorTemplate "+testAtespace+"/tmpl-a already exists") @@ -236,7 +233,6 @@ func TestActorLifecycleWithoutGoldenSnapshot(t *testing.T) { tmpl, err := tc.client.CreateActorTemplate(ctx, &ateapi.CreateActorTemplateRequest{ActorTemplate: &ateapi.ActorTemplate{ Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "no-golden"}, SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: testStorageLocation, GoldenSnapshotConfig: &ateapi.GoldenSnapshotConfig{Mode: ateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED}, }, Containers: []*ateapi.Container{{Name: "main", Image: "main@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", Command: []string{"/main"}}}, @@ -413,8 +409,7 @@ func TestCreateActorTemplate_ExternalVolumesPreview(t *testing.T) { Name: "data", External: &ateapi.ExternalVolumeSource{Template: &ateapi.ExternalVolumeTemplate{StorageClassName: "standard", CapacityBytes: 1 << 30}}, }}, - Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 1<<30)}, - SnapshotConfig: &ateapi.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, + Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 1<<30)}, }, }) if status.Code(err) != tt.wantCode { @@ -426,9 +421,8 @@ func TestCreateActorTemplate_ExternalVolumesPreview(t *testing.T) { // TestCreateActorTemplate_PerTemplateSnapshotLocationPreview pins how // snapshotConfig.storageLocation combines the PerTemplateSnapshotLocation gate -// with the client's preview opt-in. With the gate off the field is ignored, -// so any client may set it. With the gate on, only a client that opts in may -// set it; a template that leaves it empty uses the server's +// with the client's preview opt-in. Only a client that opts in may set it, and +// only with the gate on. A template that leaves it empty uses the server's // --snapshot-location. func TestCreateActorTemplate_PerTemplateSnapshotLocationPreview(t *testing.T) { ns := namespaceForTest("ns-template-storage-location-preview") @@ -442,8 +436,9 @@ func TestCreateActorTemplate_PerTemplateSnapshotLocationPreview(t *testing.T) { location string wantCode codes.Code }{ - {name: "gate off, no opt-in, location set", optIn: false, location: "gs://my-bucket/snapshots", wantCode: codes.OK}, - {name: "gate off, opt-in, location set", optIn: true, location: "gs://my-bucket/snapshots", wantCode: codes.OK}, + {name: "gate off, no opt-in, location set", optIn: false, location: "gs://my-bucket/snapshots", wantCode: codes.InvalidArgument}, + {name: "gate off, opt-in, location set", optIn: true, location: "gs://my-bucket/snapshots", wantCode: codes.InvalidArgument}, + {name: "gate off, opt-in, no location", optIn: true, wantCode: codes.OK}, {name: "gate on, no opt-in, location set", gates: []string{"PerTemplateSnapshotLocation"}, optIn: false, location: "gs://my-bucket/snapshots", wantCode: codes.InvalidArgument}, {name: "gate on, no opt-in, no location", gates: []string{"PerTemplateSnapshotLocation"}, optIn: false, wantCode: codes.OK}, {name: "gate on, opt-in, location set", gates: []string{"PerTemplateSnapshotLocation"}, optIn: true, location: "gs://my-bucket/snapshots", wantCode: codes.OK}, diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index bd49b56f88..54e7cfd48f 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -277,10 +277,9 @@ func TestCreateActor_SubstrateTemplateRef(t *testing.T) { ctx := context.Background() if _, err := tc.client.CreateActorTemplate(ctx, &ateapi.CreateActorTemplateRequest{ ActorTemplate: &ateapi.ActorTemplate{ - Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "sub-tmpl"}, - Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, - SnapshotConfig: &ateapi.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "sub-tmpl"}, + Containers: []*ateapi.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, + Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, }, }); err != nil { t.Fatalf("CreateActorTemplate failed: %v", err) @@ -401,8 +400,7 @@ func TestCreateActor_RejectsSnapshotWithExternalVolumes(t *testing.T) { defer tc.cleanup() template, err := tc.client.CreateActorTemplate(context.Background(), &ateapi.CreateActorTemplateRequest{ ActorTemplate: &ateapi.ActorTemplate{ - Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl1"}, - SnapshotConfig: &ateapi.SnapshotConfig{StorageLocation: "gs://snapshots"}, + Metadata: &ateapi.ResourceMetadata{Atespace: testAtespace, Name: "tmpl1"}, Containers: []*ateapi.Container{{ Name: "main", Image: "main@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", VolumeMounts: []*ateapi.VolumeMount{{Name: "data", MountPath: "/data"}}, }}, @@ -894,9 +892,8 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", VolumeMounts: []*ateapi.VolumeMount{{Name: "data", MountPath: tmpl.mountPath}}, }}, - Volumes: tmpl.volumes, - SnapshotConfig: &ateapi.SnapshotConfig{StorageLocation: "gs://my-bucket/snapshots"}, - Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, + Volumes: tmpl.volumes, + Resources: &ateapi.ResourceRequirements{Limits: resources.CPUMemory(1000, 512<<20)}, }, }); err != nil { t.Fatalf("CreateActorTemplate %s failed: %v", name, err) @@ -3421,7 +3418,6 @@ func createDataFidelityTemplate(t *testing.T, tc *testContext, ns string) *ateap Name: "tmpl1", }, SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: testStorageLocation, PreferredFidelity: ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, Containers: []*ateapi.Container{{ diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index d3e0937b97..0bc8ae3206 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -63,8 +63,8 @@ import ( const ( testAtespace = "test-atespace" testActorID = "id1" - // testStorageLocation is the snapshot_config.storage_location the test - // templates hand out. No object store is wired up behind it. + // testStorageLocation is the --snapshot-location the test server runs + // with. No object store is wired up behind it. testStorageLocation = "gs://fake-fake-fake" testActorJWTIssuer = "https://idp.example.test" @@ -472,9 +472,6 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st Atespace: testAtespace, Name: "tmpl1", }, - SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: testStorageLocation, - }, Containers: containers, Volumes: volumes, WorkerSelector: &ateapi.Selector{ @@ -623,9 +620,6 @@ func createTemplateWithSelector(t *testing.T, tc *testContext, name string, sele Atespace: testAtespace, Name: name, }, - SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: testStorageLocation, - }, Containers: []*ateapi.Container{ {Name: "main", Image: "main@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", Command: []string{"/main"}}, }, diff --git a/cmd/kubectl-ate/internal/cmd/actortemplate_test.go b/cmd/kubectl-ate/internal/cmd/actortemplate_test.go index 6ca43a6d9f..b7eeaeca06 100644 --- a/cmd/kubectl-ate/internal/cmd/actortemplate_test.go +++ b/cmd/kubectl-ate/internal/cmd/actortemplate_test.go @@ -158,17 +158,16 @@ func TestActorTemplateFromManifest_DemoManifests(t *testing.T) { if err != nil { t.Fatalf("reading demo manifest: %v", err) } - // ate-setup substitutes the bucket and drops the unused - // optional placeholder lines before applying. - rendered := strings.ReplaceAll(string(data), "${BUCKET_NAME}", "ate-snapshots") + // ate-setup drops the unused optional placeholder lines before + // applying. var kept []string - for _, line := range strings.Split(rendered, "\n") { + for _, line := range strings.Split(string(data), "\n") { if strings.Contains(line, "${") { continue } kept = append(kept, line) } - rendered = strings.Join(kept, "\n") + rendered := strings.Join(kept, "\n") got, err := actorTemplateFromManifest([]byte(rendered)) if err != nil { @@ -178,8 +177,12 @@ func TestActorTemplateFromManifest_DemoManifests(t *testing.T) { t.Errorf("metadata = %s/%s, want %s/%s", got.GetMetadata().GetAtespace(), got.GetMetadata().GetName(), test.atespace, test.name) } - if len(got.GetContainers()) == 0 || got.GetSnapshotConfig().GetStorageLocation() == "" { - t.Errorf("missing required fields: %v", got) + if len(got.GetContainers()) == 0 { + t.Errorf("manifest has no containers: %v", got) + } + // The API rejects storageLocation without its preview gate. + if location := got.GetSnapshotConfig().GetStorageLocation(); location != "" { + t.Errorf("manifest sets storageLocation %q, want none", location) } }) } diff --git a/demos/claude-code-multiplex/README.md b/demos/claude-code-multiplex/README.md index 6c9598b5ad..cc35f3f671 100644 --- a/demos/claude-code-multiplex/README.md +++ b/demos/claude-code-multiplex/README.md @@ -35,7 +35,8 @@ This guide assumes you know Kubernetes and the general shape of agent runtimes ( kubectl port-forward svc/api 8080:443 -n ate-system ``` - An **Anthropic API key** (the agents call Claude). -- The snapshot bucket and image registry (`BUCKET_NAME`, `KO_DOCKER_REPO`) that your Substrate install uses. The deploy step builds and pushes the workload image to that registry with a sha256-pinned reference. +- A snapshot location set at install (`SNAPSHOT_LOCATION`). The agents' snapshots are stored there. +- The image registry (`KO_DOCKER_REPO`) that your Substrate install uses. The deploy step builds and pushes the workload image to that registry with a sha256-pinned reference. - `docker buildx` (the deploy function builds the workload image — a Dockerfile-based Python + Claude Code wrapper, not a Go binary, so `ko` doesn't apply for the workload itself). ## Components @@ -59,7 +60,7 @@ ANTHROPIC_API_KEY=sk-ant-... \ go run ./cmd/ate-setup deploy demo claude-code-multiplex ``` -This creates the `claude-multiplex-demo` namespace and its 2-pod `WorkerPool`, then the `claude-multiplex-demo` atespace and three actor templates in it named `agent-luna`, `agent-mars`, `agent-orion`, and waits for each template's golden snapshot. Under the hood, the deploy function builds the workload image with `docker buildx`, pushes it to `${KO_DOCKER_REPO}/claude-multiplex-demo-workload`, resolves the pushed sha256 digest, and substitutes the digest-pinned reference plus `ANTHROPIC_API_KEY` and `BUCKET_NAME` into the agent templates at apply time. +This creates the `claude-multiplex-demo` namespace and its 2-pod `WorkerPool`, then the `claude-multiplex-demo` atespace and three actor templates in it named `agent-luna`, `agent-mars`, `agent-orion`, and waits for each template's golden snapshot. Under the hood, the deploy function builds the workload image with `docker buildx`, pushes it to `${KO_DOCKER_REPO}/claude-multiplex-demo-workload`, resolves the pushed sha256 digest, and substitutes the digest-pinned reference and `ANTHROPIC_API_KEY` into the agent templates at apply time. ### 2. Start the dashboard diff --git a/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl b/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl index 72bde3b0dc..87a32e15fe 100644 --- a/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl +++ b/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl @@ -48,4 +48,3 @@ resources: memoryBytes: 2147483648 # 2Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/claude-multiplex-demo/ diff --git a/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl b/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl index 96b0ec95de..37c2cb07f6 100644 --- a/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl +++ b/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl @@ -40,4 +40,3 @@ resources: memoryBytes: 2147483648 # 2Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/claude-multiplex-demo/ diff --git a/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl b/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl index ce3240cda2..d1fc8adfea 100644 --- a/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl +++ b/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl @@ -40,4 +40,3 @@ resources: memoryBytes: 2147483648 # 2Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/claude-multiplex-demo/ diff --git a/demos/counter/README.md b/demos/counter/README.md index 604b75dbe5..3f99ba6316 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -10,6 +10,7 @@ The demo uses two kinds of resources: the `WorkerPool` is a Kubernetes CRD, whil - A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) (with `ATE_INSTALL_KIND=true` exported in your shell) or [GKE Quickstart](../../README.md#gke-quickstart-development). +- A snapshot location set at install (`SNAPSHOT_LOCATION`). The demo stores its snapshots there. ## How to Run on Agent Substrate @@ -21,9 +22,8 @@ it, skip to [How to Use](#how-to-use). > [!NOTE] > Do not manually edit `demos/counter/counter.yaml.tmpl` or -> `demos/counter/counter-template.yaml.tmpl`. `ate-setup` -> automatically injects your `${BUCKET_NAME}` environment variable during -> deployment. +> `demos/counter/counter-template.yaml.tmpl`. `ate-setup` fills in their +> `${...}` placeholders during deployment.
Installation on MicroVM diff --git a/demos/counter/counter-microvm-csi-test-template.yaml b/demos/counter/counter-microvm-csi-test-template.yaml index 4b74997769..da6be1bfb3 100644 --- a/demos/counter/counter-microvm-csi-test-template.yaml +++ b/demos/counter/counter-microvm-csi-test-template.yaml @@ -40,7 +40,6 @@ resources: memoryBytes: 536870912 # 512Mi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES - storageLocation: gs://ate-snapshots/ate-demo-counter-microvm-csi/ volumes: - name: data external: diff --git a/demos/counter/counter-microvm-template.yaml.tmpl b/demos/counter/counter-microvm-template.yaml.tmpl index 9e883f6d82..208f96885a 100644 --- a/demos/counter/counter-microvm-template.yaml.tmpl +++ b/demos/counter/counter-microvm-template.yaml.tmpl @@ -62,7 +62,6 @@ snapshotConfig: # demo promises. ROOTFS is also available on this class: it would keep the # file counter and any rootfs writes but cold-boot the memory counter. preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/ate-demo-counter-microvm/ volumes: - name: data durableDir: {} diff --git a/demos/counter/counter-template.yaml.tmpl b/demos/counter/counter-template.yaml.tmpl index b92c17ee2a..52ec23b437 100644 --- a/demos/counter/counter-template.yaml.tmpl +++ b/demos/counter/counter-template.yaml.tmpl @@ -57,7 +57,6 @@ snapshotConfig: # memory cold-boots) is exercised by the e2e demo suite's own # parameterized templates, not by this one. preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/ate-demo-counter/ volumes: - name: data durableDir: {} diff --git a/demos/egress/egress-microvm-template.yaml.tmpl b/demos/egress/egress-microvm-template.yaml.tmpl index 6c21947f10..136c3bf2e9 100644 --- a/demos/egress/egress-microvm-template.yaml.tmpl +++ b/demos/egress/egress-microvm-template.yaml.tmpl @@ -69,4 +69,3 @@ resources: memoryBytes: 536870912 # 512Mi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/ate-demo-egress-microvm/ diff --git a/demos/egress/egress-template.yaml.tmpl b/demos/egress/egress-template.yaml.tmpl index 0815d4aeb4..b6987fc163 100644 --- a/demos/egress/egress-template.yaml.tmpl +++ b/demos/egress/egress-template.yaml.tmpl @@ -61,4 +61,3 @@ resources: memoryBytes: 536870912 # 512Mi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/ate-demo-egress/ diff --git a/demos/jupyter/README.md b/demos/jupyter/README.md index 2fdd7f5faf..99981cf81a 100644 --- a/demos/jupyter/README.md +++ b/demos/jupyter/README.md @@ -6,6 +6,7 @@ This demo runs a Jupyter Notebook on Agent Substrate. It demonstrates how standa - A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) (with `ATE_INSTALL_KIND=true` exported in your shell) or [GKE Quickstart](../../README.md#gke-quickstart-development). +- A snapshot location set at install (`SNAPSHOT_LOCATION`). The demo stores its snapshots there. ## Deploy the Jupyter Notebook Demo diff --git a/demos/jupyter/jupyter-template.yaml.tmpl b/demos/jupyter/jupyter-template.yaml.tmpl index 239fa4b022..df8cd69396 100644 --- a/demos/jupyter/jupyter-template.yaml.tmpl +++ b/demos/jupyter/jupyter-template.yaml.tmpl @@ -47,4 +47,3 @@ resources: memoryBytes: 4294967296 # 4Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/ate-demo-jupyter/ diff --git a/demos/multi-template/README.md b/demos/multi-template/README.md index e92057b743..0fb0f14185 100644 --- a/demos/multi-template/README.md +++ b/demos/multi-template/README.md @@ -11,16 +11,12 @@ or namespace. - A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) (with `ATE_INSTALL_KIND=true` exported in your shell) or [GKE Quickstart](../../README.md#gke-quickstart-development). +- A snapshot location set at install (`SNAPSHOT_LOCATION`). The demo stores its snapshots there. ## How to Run on Agent Substrate ### 1. Build and Deploy -> [!NOTE] -> Do not manually edit the `demos/multi-template/*.yaml.tmpl` manifests. -> `ate-setup` automatically injects your `${BUCKET_NAME}` environment -> variable during deployment. - ```bash go run ./cmd/ate-setup deploy demo multi-template ``` diff --git a/demos/multi-template/counter-template.yaml.tmpl b/demos/multi-template/counter-template.yaml.tmpl index e689b1f599..be2a8c6a07 100644 --- a/demos/multi-template/counter-template.yaml.tmpl +++ b/demos/multi-template/counter-template.yaml.tmpl @@ -35,4 +35,3 @@ resources: memoryBytes: 536870912 # 512Mi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/ate-demo-multi-template-counter/ diff --git a/demos/multi-template/fspersist-template.yaml.tmpl b/demos/multi-template/fspersist-template.yaml.tmpl index 1b573243f2..8fffb50648 100644 --- a/demos/multi-template/fspersist-template.yaml.tmpl +++ b/demos/multi-template/fspersist-template.yaml.tmpl @@ -35,4 +35,3 @@ resources: memoryBytes: 536870912 # 512Mi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/ate-demo-multi-template-fspersist/ diff --git a/demos/parking/README.md b/demos/parking/README.md index 74b5c26487..53bf696ffa 100644 --- a/demos/parking/README.md +++ b/demos/parking/README.md @@ -15,16 +15,12 @@ See [docs/request-parking.md](../../docs/request-parking.md) for the design. - A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) (with `ATE_INSTALL_KIND=true` exported in your shell) or [GKE Quickstart](../../README.md#gke-quickstart-development). +- A snapshot location set at install (`SNAPSHOT_LOCATION`). The demo stores its snapshots there. ## How to Run on Agent Substrate ### 1. Build and Deploy -> [!NOTE] -> Do not manually edit the `demos/parking/*.yaml.tmpl` manifests. `ate-setup` -> automatically injects your `${BUCKET_NAME}` environment variable during -> deployment. - ```bash go run ./cmd/ate-setup deploy demo parking ``` diff --git a/demos/parking/parking-template.yaml.tmpl b/demos/parking/parking-template.yaml.tmpl index 4753f7a02a..f92a6df94d 100644 --- a/demos/parking/parking-template.yaml.tmpl +++ b/demos/parking/parking-template.yaml.tmpl @@ -34,4 +34,3 @@ resources: memoryBytes: 536870912 # 512Mi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/ate-demo-parking/ diff --git a/demos/sandbox/README.md b/demos/sandbox/README.md index b4d87af47c..3a5b289a67 100644 --- a/demos/sandbox/README.md +++ b/demos/sandbox/README.md @@ -16,15 +16,13 @@ It allows you to run arbitrary commands in an sandboxed, isolated container (run - A k8s cluster with Agent Substrate installed as in the [Quickstart](../../README.md#quickstart-development) (with `ATE_INSTALL_KIND=true` exported in your shell) or [GKE Quickstart](../../README.md#gke-quickstart-development). +- A snapshot location set at install (`SNAPSHOT_LOCATION`). The demo stores its snapshots there. - `kubectl-ate` CLI installed (can be installed via `go install ./cmd/kubectl-ate`). ## How to Run on Agent Substrate ### 1. Build and Deploy -> [!NOTE] -> Do not manually edit the `demos/sandbox/*.yaml.tmpl` manifests. `ate-setup` automatically injects your `${BUCKET_NAME}` environment variable during deployment. - Use `ate-setup` to build the image and deploy the demo to your cluster: ```bash diff --git a/demos/sandbox/manual-test-multi-template.yaml b/demos/sandbox/manual-test-multi-template.yaml index 5399ed7ead..2b0489e4a0 100644 --- a/demos/sandbox/manual-test-multi-template.yaml +++ b/demos/sandbox/manual-test-multi-template.yaml @@ -42,7 +42,6 @@ resources: memoryBytes: 536870912 # 512Mi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://ate-snapshots/ate-manual-test-multi/ volumes: - name: vol-hostpath external: diff --git a/demos/sandbox/sandbox-template.yaml.tmpl b/demos/sandbox/sandbox-template.yaml.tmpl index 358d32b294..57fb1aa382 100644 --- a/demos/sandbox/sandbox-template.yaml.tmpl +++ b/demos/sandbox/sandbox-template.yaml.tmpl @@ -36,4 +36,3 @@ resources: memoryBytes: 1073741824 # 1Gi snapshotConfig: preferredFidelity: SNAPSHOT_FIDELITY_MEMORY - storageLocation: gs://${BUCKET_NAME}/ate-demo-sandbox/ diff --git a/docs/api-guide.md b/docs/api-guide.md index bd45a67223..7fba29d68a 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -101,7 +101,7 @@ The `ActorTemplate` defines the code, environment, and state-management policies | :--- | :--- | :--- | | `containers` | `[]Container` | **Required.** The workload definition — see [Container Fields](#container-fields) below. Each container may also declare an optional `wakeupProbe` HTTP probe — see [Container Wakeup Probe](#container-wakeup-probe-wakeupprobe). | | `workerSelector` | `*Selector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels (`matchLabels`). If unset, all pools are eligible (subject to the actor's own `worker_selector`). The pool decides the sandbox class, so this is also how a template pins one: if your cluster has pools of more than one sandbox class and the workload needs a particular class, make sure the selector points only at pools of that class. An actor booting with no snapshot and no selector may land on any class, and keeps it from its first checkpoint on. | -| `snapshotConfig` | `SnapshotConfig` | Optional; omitted fields take their defaults. The base object-storage location snapshots are written under (a preview field, ignored without the `PerTemplateSnapshotLocation` gate), the snapshot fidelity (`preferredFidelity`) every Suspend captures, and automated golden snapshot creation (`goldenSnapshotConfig.mode`). See [Snapshot Storage Layout](#snapshot-storage-layout). | +| `snapshotConfig` | `SnapshotConfig` | Optional; omitted fields take their defaults. The base object-storage location snapshots are written under (a preview field that requires the `PerTemplateSnapshotLocation` gate), the snapshot fidelity (`preferredFidelity`) every Suspend captures, and automated golden snapshot creation (`goldenSnapshotConfig.mode`). See [Snapshot Storage Layout](#snapshot-storage-layout). | | `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each a `durableDir`, an `external` CSI volume source (see [CSI Volumes Guide](csi-volumes.md)), or a `systemInfo` volume (see [SystemInfo Volumes](#systeminfo-volumes)). Every declared volume must be mounted by at least one container. A `durableDir` is created `0777`, as Kubernetes creates an emptyDir, so a container running as any user can write it. A snapshot restores it with the mode it had, so the container's own chmod survives suspend and resume. | | `resources` | `*ResourceRequirements` | Required. Declares each actor's compute size via `limits`, which must set both `cpuMillicores` and `memoryBytes` — see [Sandbox Right-Sizing](#sandbox-right-sizing-resources). Immutable, like the rest of the template. | | `goldenEgressPolicy` | `EgressPolicyTemplate` | **Preview.** Optional egress policy for the golden actor during initialization. Requires at least one rule, using the same rule defaults and validation as `EgressPolicy`. See [Golden Egress Policy](#golden-egress-policy-goldenegresspolicy). | @@ -395,8 +395,6 @@ resources: limits: cpuMillicores: 1000 memoryBytes: 1073741824 # 1Gi -snapshotConfig: - storageLocation: gs://my-bucket/secret-agent ``` ### Snapshot Storage Layout @@ -409,13 +407,13 @@ snapshotConfig: ``` > [!NOTE] -> Snapshots are stored under ate-api-server's cluster-wide `--snapshot-location`. `snapshotConfig.storageLocation` is an optional preview field: with the `PerTemplateSnapshotLocation` preview gate (`--preview=PerTemplateSnapshotLocation`), a template that sets it stores its snapshots there instead. Without the gate, the field is ignored. `` above is whichever of the two applies. +> Snapshots are stored under ate-api-server's cluster-wide `--snapshot-location`. `snapshotConfig.storageLocation` is an optional preview field: with the `PerTemplateSnapshotLocation` preview gate (`--preview=PerTemplateSnapshotLocation`), a template that sets it stores its snapshots there instead. `` above is whichever of the two applies. > -> With the gate enabled, only a client that opts in to preview features (`kubectl ate --preview`, or the `Ate-Preview-Allowed: true` header) may set `storageLocation`. `CreateActorTemplate` fails with `FailedPrecondition` when the template leaves it empty and ate-api-server has no `--snapshot-location`. +> Only a client that opts in to preview features (`kubectl ate --preview`, or the `Ate-Preview-Allowed: true` header) may set `storageLocation`, and only with the gate enabled. Otherwise the request fails with `InvalidArgument`. A template stored before the API enforced this keeps the field, and ate-api-server ignores it while the gate is off. `CreateActorTemplate` fails with `FailedPrecondition` when the template leaves it empty and ate-api-server has no `--snapshot-location`. > > Changing the flag or toggling the gate moves `` for every template it affects. Snapshots already taken stay where they are and can still be resumed from. An actor's next suspend writes under the new location and releases its previous snapshot from the old one. Until then, the actor cannot change its `actor_template` (see `UpdateActor` below), and deleting it collects its prefix under the old location. If that release fails, the previous snapshot's objects stay under the old location, and deleting the actor later does not collect them. -The objects of a snapshot (its manifest, memory image, durable-data tar) are named below it. So for the template above, a snapshot of an actor in atespace `team-a` is stored at `gs://my-bucket/secret-agent/atespaces/team-a/actors/3f8b…/snapshots/f47ac10b-…`, and the template's golden snapshot under `gs://my-bucket/secret-agent/atespaces/team-a/tags/`. +The objects of a snapshot (its manifest, memory image, durable-data tar) are named below it. So with `--snapshot-location=gs://my-bucket/`, a snapshot of an actor in atespace `team-a` is stored at `gs://my-bucket/atespaces/team-a/actors/3f8b…/snapshots/f47ac10b-…`, and a golden snapshot under `gs://my-bucket/atespaces/team-a/tags/`. An actor takes a series of snapshots over its life, so it gets a prefix of its own and each snapshot sits below it. A tag holds exactly one, so the tag's @@ -522,7 +520,7 @@ Each entry of `versions` (`SandboxVersionConfig`): | :--- | :--- | :--- | | `name` | `string` | **Required.** Identifies the version within the config. A DNS label (lower-case alphanumerics and `-`, at most 63 characters). | | `state` | `string` | Optional. `Enabled` (default) or `Disabled`. The version named by `defaultVersion` cannot be `Disabled`. | -| `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A microVM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, is unreachable, or fails with an error it does not recognize, such as GCS refusing its credentials for a public bucket) is then read anonymously, which serves public buckets such as `gs://gvisor`; any other refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (an `ActorTemplate`'s `snapshotConfig.location` followed by `/atespaces/`), so assets must not be stored there. On GKE, setup-gcp lets the plugin read only the asset bucket, `-assets`, so keep assets there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | +| `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A microVM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, is unreachable, or fails with an error it does not recognize, such as GCS refusing its credentials for a public bucket) is then read anonymously, which serves public buckets such as `gs://gvisor`; any other refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (a snapshot location followed by `/atespaces/`), so assets must not be stored there. On GKE, setup-gcp lets the plugin read only the asset bucket, `-assets`, so keep assets there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | A cluster-wide gVisor `SandboxConfig` (`gvisor-default`, annotated as the class default) is installed with the platform, so gVisor `WorkerPool`s use it automatically without any extra setup. diff --git a/docs/csi-volumes.md b/docs/csi-volumes.md index d44fb43762..932afe8f84 100644 --- a/docs/csi-volumes.md +++ b/docs/csi-volumes.md @@ -192,8 +192,6 @@ resources: limits: cpuMillicores: 1000 memoryBytes: 1073741824 # 1Gi -snapshotConfig: - storageLocation: gs://my-snapshots-bucket/stateful-agent volumes: - name: shared-storage external: diff --git a/docs/operator-install.md b/docs/operator-install.md index 225610d191..2cec3c3054 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -325,7 +325,9 @@ ate-api-server stores every actor and tag snapshot under a cluster-wide `--snapshot-location`, an object-storage URI such as `gs://BUCKET/PREFIX/`. Set it with `SNAPSHOT_LOCATION` (or ate-setup's `--snapshot-location`). It has no default, and ate-setup fails before deploying without it unless the -`PerTemplateSnapshotLocation` preview gate is enabled. If you deploy ate-api-server +`PerTemplateSnapshotLocation` preview gate is enabled. Even with the gate, the +demos and benchmarks need it, since they do not set +`snapshotConfig.storageLocation`. So do e2e runs, which deploy the demos. If you deploy ate-api-server some other way, pass the flag yourself; without the gate, the server refuses to start without it. @@ -363,7 +365,7 @@ With the `PerTemplateSnapshotLocation` preview gate, an ActorTemplate may set that leave it empty use `--snapshot-location`. Grant ate-api-server's identity and the snapshot transfer service account object admin and bucket viewer on any other bucket a template names, as setup-gcp does on the snapshot bucket. Without the gate, -ate-api-server ignores `snapshotConfig.storageLocation`. See +ate-api-server rejects templates that set `snapshotConfig.storageLocation`. See [Snapshot Storage Layout](api-guide.md#snapshot-storage-layout). ## Ingress to actors diff --git a/internal/e2e/fixture.go b/internal/e2e/fixture.go index 391b544122..5410bb7c0c 100644 --- a/internal/e2e/fixture.go +++ b/internal/e2e/fixture.go @@ -32,11 +32,10 @@ import ( // ActorTemplate fixture templates carry: the substrate counterpart of // fixtureSubstitutions, whose block fragments are CRD-shaped and now serve // only the pool manifests. The inline placeholders match the pool manifests' -// values so one (bucket, name) pair renders both halves of a fixture. +// values so one name renders both halves of a fixture. // trustBundles names the trust bundles projected into one trust-bundle.pem. -func substrateTemplateSubstitutions(bucket, name string, trustBundles []string) (inline, blocks map[string]string) { +func substrateTemplateSubstitutions(name string, trustBundles []string) (inline, blocks map[string]string) { inline = map[string]string{ - "${BUCKET_NAME}": bucket, "${FIXTURE_SUFFIX}": "-" + name, } blocks = map[string]string{ @@ -134,7 +133,7 @@ type SubstrateFixtureManifests struct { // // Returns the fixture's atespace (the same string that names the k8s // namespace holding the pool) and the created templates. -func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients, manifests SubstrateFixtureManifests, bucket, name string, trustBundles []string, opts ...FixtureOption) (string, []*ateapi.ActorTemplate) { +func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients, manifests SubstrateFixtureManifests, name string, trustBundles []string, opts ...FixtureOption) (string, []*ateapi.ActorTemplate) { t.Helper() var cfg fixtureConfig @@ -145,7 +144,7 @@ func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients, // The pool manifest also carries the namespace. Its cleanup is registered // first so it runs last (t.Cleanup is LIFO), after the templates that // select the pool's workers are gone. - inline, blocks := fixtureSubstitutions(bucket, name) + inline, blocks := fixtureSubstitutions(name) poolManifest := renderManifest(t, manifests.Pool, inline, blocks) koApply(t, poolManifest) t.Cleanup(func() { @@ -156,7 +155,7 @@ func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients, RunCmd(t, "kubectl", delArgs...) }) - tmplInline, tmplBlocks := substrateTemplateSubstitutions(bucket, name, trustBundles) + tmplInline, tmplBlocks := substrateTemplateSubstitutions(name, trustBundles) maps.Copy(tmplInline, cfg.templateValues) rendered := koResolve(t, renderManifest(t, manifests.Template, tmplInline, tmplBlocks)) templates := decodeSubstrateTemplates(t, rendered) @@ -166,6 +165,13 @@ func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients, if got := tmpl.GetMetadata().GetAtespace(); got != atespace { t.Fatalf("fixture %s declares templates in different atespaces (%q and %q)", manifests.Template, atespace, got) } + location := SnapshotLocation(t, "ate-e2e-"+tmpl.GetMetadata().GetName()+tmplInline["${FIXTURE_SUFFIX}"]) + if location != "" { + if tmpl.SnapshotConfig == nil { + tmpl.SnapshotConfig = &ateapi.SnapshotConfig{} + } + tmpl.SnapshotConfig.StorageLocation = location + } } if _, err := clients.SubstrateAPI.CreateAtespace(ctx, &ateapi.CreateAtespaceRequest{Atespace: &ateapi.Atespace{Metadata: &ateapi.ResourceMetadata{Name: atespace}}}); err != nil && status.Code(err) != codes.AlreadyExists { diff --git a/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl b/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl index 549ec8cdc9..494189840b 100644 --- a/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl +++ b/internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl @@ -40,8 +40,6 @@ resources: limits: cpuMillicores: 1000 memoryBytes: 536870912 # 512Mi -snapshotConfig: - storageLocation: gs://${BUCKET_NAME}/ate-e2e-caps-default${FIXTURE_SUFFIX}/ --- @@ -70,8 +68,6 @@ resources: limits: cpuMillicores: 1000 memoryBytes: 536870912 # 512Mi -snapshotConfig: - storageLocation: gs://${BUCKET_NAME}/ate-e2e-caps-exact${FIXTURE_SUFFIX}/ --- # No capabilities at all, and the probe still binds :80: both runtimes lift @@ -98,5 +94,3 @@ resources: limits: cpuMillicores: 1000 memoryBytes: 536870912 # 512Mi -snapshotConfig: - storageLocation: gs://${BUCKET_NAME}/ate-e2e-caps-none${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/probe/probe-bootfetch-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-bootfetch-template.yaml.tmpl index 576976fe96..81cc2ca883 100644 --- a/internal/e2e/fixtures/probe/probe-bootfetch-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-bootfetch-template.yaml.tmpl @@ -63,5 +63,3 @@ resources: limits: cpuMillicores: 1000 memoryBytes: 536870912 # 512Mi -snapshotConfig: - storageLocation: gs://${BUCKET_NAME}/ate-e2e-probe-bootfetch${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl index 55138cdd5a..52d7ab1637 100644 --- a/internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl @@ -50,5 +50,3 @@ resources: limits: cpuMillicores: 1000 memoryBytes: 268435456 # 256Mi -snapshotConfig: - storageLocation: gs://${BUCKET_NAME}/ate-e2e${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl index 8b4270db98..c3b1f452a5 100644 --- a/internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl @@ -48,5 +48,3 @@ resources: limits: cpuMillicores: 1000 memoryBytes: 536870912 # 512Mi -snapshotConfig: - storageLocation: gs://${BUCKET_NAME}/ate-e2e${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl index 3fb12d26c3..c1004a20b6 100644 --- a/internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl @@ -42,5 +42,3 @@ resources: limits: cpuMillicores: 2000 memoryBytes: 536870912 # 512Mi -snapshotConfig: - storageLocation: gs://${BUCKET_NAME}/ate-e2e${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/probe/probe-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-template.yaml.tmpl index 6d7c432c6b..93fb703c0a 100644 --- a/internal/e2e/fixtures/probe/probe-template.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe-template.yaml.tmpl @@ -60,5 +60,3 @@ resources: limits: cpuMillicores: 1000 memoryBytes: 536870912 # 512Mi -snapshotConfig: - storageLocation: gs://${BUCKET_NAME}/ate-e2e-probe${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/security/atespace-template.yaml.tmpl b/internal/e2e/fixtures/security/atespace-template.yaml.tmpl index d6d6464605..6817e69b3d 100644 --- a/internal/e2e/fixtures/security/atespace-template.yaml.tmpl +++ b/internal/e2e/fixtures/security/atespace-template.yaml.tmpl @@ -74,5 +74,3 @@ resources: limits: cpuMillicores: 1000 memoryBytes: 536870912 # 512Mi -snapshotConfig: - storageLocation: gs://${BUCKET_NAME}/ate-e2e${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/testserver/grpcecho-template.yaml.tmpl b/internal/e2e/fixtures/testserver/grpcecho-template.yaml.tmpl index 717868e298..ea97c6da4a 100644 --- a/internal/e2e/fixtures/testserver/grpcecho-template.yaml.tmpl +++ b/internal/e2e/fixtures/testserver/grpcecho-template.yaml.tmpl @@ -49,5 +49,3 @@ resources: limits: cpuMillicores: 1000 memoryBytes: 536870912 # 512Mi -snapshotConfig: - storageLocation: gs://${BUCKET_NAME}/ate-e2e-grpcecho${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl b/internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl index 9663ed2f37..694277c16c 100644 --- a/internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl +++ b/internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl @@ -35,5 +35,3 @@ resources: limits: cpuMillicores: 1000 memoryBytes: 536870912 # 512Mi -snapshotConfig: - storageLocation: gs://${BUCKET_NAME}/ate-e2e-websocket${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/preview.go b/internal/e2e/preview.go index fb997c64c2..21d4be23b2 100644 --- a/internal/e2e/preview.go +++ b/internal/e2e/preview.go @@ -73,6 +73,22 @@ func previewDialOptions() []grpc.DialOption { return nil } +// SnapshotLocation returns gs://$BUCKET_NAME/prefix/ when the install under +// test enabled the PerTemplateSnapshotLocation gate, and "" otherwise. Tests +// set it as a template's storageLocation, which the API rejects without the +// gate, so the gate-off lane stores snapshots under --snapshot-location. +func SnapshotLocation(t testing.TB, prefix string) string { + t.Helper() + if !preview.IsEnabled(preview.GatePerTemplateSnapshotLocation) { + return "" + } + env, err := CheckEnv("BUCKET_NAME") + if err != nil { + t.Fatalf("CheckEnv: %v", err) + } + return "gs://" + env["BUCKET_NAME"] + "/" + prefix + "/" +} + // RequirePreview skips t unless the install under test enabled gate. Tests of // a preview feature call it first, so a run without the gate skips them. func RequirePreview(t testing.TB, gate preview.Gate) { diff --git a/internal/e2e/preview_test.go b/internal/e2e/preview_test.go index 0efb5b7f08..5bab9a6a77 100644 --- a/internal/e2e/preview_test.go +++ b/internal/e2e/preview_test.go @@ -19,8 +19,29 @@ import ( "testing" "k8s.io/utils/ptr" + + "go.ate.dev/substrate/internal/preview" ) +func TestSnapshotLocation(t *testing.T) { + t.Setenv("BUCKET_NAME", "test-bucket") + for _, tc := range []struct { + name string + gates []string + want string + }{ + {name: "gate off"}, + {name: "gate on", gates: []string{string(preview.GatePerTemplateSnapshotLocation)}, want: "gs://test-bucket/ate-e2e-probe/"}, + } { + t.Run(tc.name, func(t *testing.T) { + preview.InitForTest(t, tc.gates...) + if got := SnapshotLocation(t, "ate-e2e-probe"); got != tc.want { + t.Errorf("SnapshotLocation() = %q, want %q", got, tc.want) + } + }) + } +} + func TestPreviewGates(t *testing.T) { for _, tc := range []struct { name string diff --git a/internal/e2e/probe.go b/internal/e2e/probe.go index 256ba0ff7a..ce0f7b6682 100644 --- a/internal/e2e/probe.go +++ b/internal/e2e/probe.go @@ -68,7 +68,7 @@ func WithSystemRoots() ProbeOption { // out from under another running concurrently. It returns the fixture's // atespace (which also names the k8s namespace holding the pool) and the // created ActorTemplate, already golden-snapshotted. -func DeployProbe(t *testing.T, bucket, name string, opts ...ProbeOption) (string, *ateapi.ActorTemplate) { +func DeployProbe(t *testing.T, name string, opts ...ProbeOption) (string, *ateapi.ActorTemplate) { t.Helper() var cfg probeConfig @@ -81,6 +81,6 @@ func DeployProbe(t *testing.T, bucket, name string, opts ...ProbeOption) (string EnsureEgressTrustBundle(t, context.Background(), GetClients()) } - atespace, templates := DeploySubstrateFixture(t, context.Background(), GetClients(), probeManifests, bucket, name, cfg.trustBundles) + atespace, templates := DeploySubstrateFixture(t, context.Background(), GetClients(), probeManifests, name, cfg.trustBundles) return atespace, templates[0] } diff --git a/internal/e2e/probe_test.go b/internal/e2e/probe_test.go index a610e1f37e..c3810ab657 100644 --- a/internal/e2e/probe_test.go +++ b/internal/e2e/probe_test.go @@ -45,7 +45,7 @@ func TestProbeTemplate_TrustBundle(t *testing.T) { // Strict decoding is what proves the fragment landed at the right // depth: misindented, it would fail the protojson decode or parse // as some other field. - inline, blocks := substrateTemplateSubstitutions("test-bucket", "render", cfg.trustBundles) + inline, blocks := substrateTemplateSubstitutions("render", cfg.trustBundles) rendered, err := os.ReadFile(renderManifest(t, probeManifests.Template, inline, blocks)) if err != nil { t.Fatalf("reading rendered manifest: %v", err) diff --git a/internal/e2e/sandbox.go b/internal/e2e/sandbox.go index 387e409058..234be0bf93 100644 --- a/internal/e2e/sandbox.go +++ b/internal/e2e/sandbox.go @@ -166,21 +166,20 @@ func TemplateReadyTimeout(t *testing.T) time.Duration { // One template serves both sandbox classes so the two variants of a fixture // cannot drift apart. See renderManifest for the placeholder kinds a template // can carry. -func RenderFixtureManifest(t *testing.T, relPath, bucket, name string) string { +func RenderFixtureManifest(t *testing.T, relPath, name string) string { t.Helper() - inline, blocks := fixtureSubstitutions(bucket, name) + inline, blocks := fixtureSubstitutions(name) return renderManifest(t, relPath, inline, blocks) } // fixtureSubstitutions is the placeholder set the internal/e2e/fixtures // manifest templates carry, split into the inline and whole-line-block kinds // RenderFixtureManifest treats differently. -func fixtureSubstitutions(bucket, name string) (inline, blocks map[string]string) { +func fixtureSubstitutions(name string) (inline, blocks map[string]string) { inline = map[string]string{ - "${BUCKET_NAME}": bucket, "${WORKER_IMAGE}": "ko://go.ate.dev/substrate/cmd/ateworker-gvisor", // The manifest-side half of FixtureName: it suffixes the fixture's - // namespace, and with it the snapshot prefix underneath. + // namespace. "${FIXTURE_SUFFIX}": "-" + name, } blocks = map[string]string{ diff --git a/internal/e2e/sandbox_test.go b/internal/e2e/sandbox_test.go index 9ac345e2be..427bb5d823 100644 --- a/internal/e2e/sandbox_test.go +++ b/internal/e2e/sandbox_test.go @@ -79,7 +79,7 @@ var substrateFixtures = []struct { // a WorkerPool that never gets a micro-VM worker. func renderPool(t *testing.T, relPath string) *atev1.WorkerPool { t.Helper() - raw, err := os.ReadFile(RenderFixtureManifest(t, relPath, "test-bucket", "render")) + raw, err := os.ReadFile(RenderFixtureManifest(t, relPath, "render")) if err != nil { t.Fatalf("reading the rendered %s: %v", relPath, err) } @@ -115,7 +115,7 @@ func renderPool(t *testing.T, relPath string) *atev1.WorkerPool { // the cluster as literal text. func renderTemplates(t *testing.T, relPath string, values map[string]string) []*ateapi.ActorTemplate { t.Helper() - inline, blocks := substrateTemplateSubstitutions("test-bucket", "render", nil) + inline, blocks := substrateTemplateSubstitutions("render", nil) maps.Copy(inline, values) rendered, err := os.ReadFile(renderManifest(t, relPath, inline, blocks)) if err != nil { @@ -149,14 +149,12 @@ func TestRenderSubstrateFixtures_GVisor(t *testing.T) { } for _, tmpl := range templates { name := tmpl.GetMetadata().GetName() - // An inline placeholder with an empty value must substitute, not - // delete its line: the location is what the golden snapshot needs. - location := tmpl.GetSnapshotConfig().GetStorageLocation() - if want := "gs://test-bucket/"; !strings.HasPrefix(location, want) { - t.Errorf("template %s snapshot location = %q, want it to start with %q", name, location, want) + atespace := tmpl.GetMetadata().GetAtespace() + if want := "-render"; !strings.HasSuffix(atespace, want) { + t.Errorf("template %s atespace = %q, want it to end with %q", name, atespace, want) } - if strings.Contains(location, "-microvm") { - t.Errorf("template %s snapshot location = %q, want no micro-VM suffix", name, location) + if strings.Contains(atespace, "-microvm") { + t.Errorf("template %s atespace = %q, want no micro-VM suffix", name, atespace) } // The selector is what ties the template to its fixture's pool. for k, v := range tmpl.GetWorkerSelector().GetMatchLabels() { @@ -170,7 +168,7 @@ func TestRenderSubstrateFixtures_GVisor(t *testing.T) { } // TestRenderSubstrateFixtures_MicroVM pins the micro-VM rendering: the -// templates carry limits and the snapshots land under their own prefix. +// templates carry limits and land in their own atespace. func TestRenderSubstrateFixtures_MicroVM(t *testing.T) { t.Setenv(sandboxClassEnv, SandboxClassMicroVM) for _, fixture := range substrateFixtures { @@ -193,9 +191,9 @@ func TestRenderSubstrateFixtures_MicroVM(t *testing.T) { if tmpl.GetResources().GetLimits().GetMemoryBytes() == 0 { t.Errorf("template %s declares no memory limit", name) } - location := tmpl.GetSnapshotConfig().GetStorageLocation() - if want := "-microvm-render/"; !strings.HasSuffix(location, want) { - t.Errorf("template %s snapshot location = %q, want it to end with %q", name, location, want) + atespace := tmpl.GetMetadata().GetAtespace() + if want := "-microvm-render"; !strings.HasSuffix(atespace, want) { + t.Errorf("template %s atespace = %q, want it to end with %q", name, atespace, want) } } }) diff --git a/internal/e2e/suites/authz/authz_test.go b/internal/e2e/suites/authz/authz_test.go index f6c9709833..87b914c5bf 100644 --- a/internal/e2e/suites/authz/authz_test.go +++ b/internal/e2e/suites/authz/authz_test.go @@ -34,12 +34,11 @@ import ( // against it. The subtests run in order, not in parallel: each sets the // fixture atespace's AccessPolicy for its own principals. func TestAuthz(t *testing.T) { - env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") - if err != nil { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { t.Fatalf("CheckEnv failed: %v", err) } ctx := context.Background() - f := deployFixture(t, ctx, env["BUCKET_NAME"]) + f := deployFixture(t, ctx) t.Run("BasicFlow", func(t *testing.T) { testBasicFlow(t, ctx, f) }) t.Run("SecurityRegression", func(t *testing.T) { testSecurityRegression(t, ctx, f) }) diff --git a/internal/e2e/suites/authz/helpers_test.go b/internal/e2e/suites/authz/helpers_test.go index f7d3137d28..5c81dbfd30 100644 --- a/internal/e2e/suites/authz/helpers_test.go +++ b/internal/e2e/suites/authz/helpers_test.go @@ -277,10 +277,10 @@ func deleteAtespace(ctx context.Context, clients *e2e.Clients, name string) erro // deployFixture deploys the probe template into a new atespace and creates // one actor from it, all as the bootstrap owner. The actor is never resumed: // the tests only need an actor that exists. -func deployFixture(t *testing.T, ctx context.Context, bucket string) *fixture { +func deployFixture(t *testing.T, ctx context.Context) *fixture { t.Helper() clients := e2e.GetClients() - atespace, _ := e2e.DeployProbe(t, bucket, "authz") + atespace, _ := e2e.DeployProbe(t, "authz") actor := &ateapi.ObjectRef{Atespace: atespace, Name: actorName} template := &ateapi.ObjectRef{Atespace: atespace, Name: e2e.ProbeName} if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapi.CreateActorRequest{Actor: &ateapi.Actor{ diff --git a/internal/e2e/suites/capabilities/capabilities_test.go b/internal/e2e/suites/capabilities/capabilities_test.go index 0f03febd7f..7c02a1c3a2 100644 --- a/internal/e2e/suites/capabilities/capabilities_test.go +++ b/internal/e2e/suites/capabilities/capabilities_test.go @@ -52,14 +52,13 @@ type capabilitiesResponse struct { // It runs against whichever sandbox class E2E_SANDBOX_CLASS selects, so CI // covers both gvisor and micro-VM from one suite. func TestActorCapabilities(t *testing.T) { - env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") - if err != nil { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { t.Fatalf("CheckEnv failed: %v", err) } ctx := context.Background() clients := e2e.GetClients() - namespace := deployFixture(t, ctx, clients, env["BUCKET_NAME"]) + namespace := deployFixture(t, ctx, clients) tests := []struct { name string @@ -145,12 +144,12 @@ func assertSameCapabilities(t *testing.T, set string, got, want []string) { // container cannot start — for example because a needed capability was // dropped — fails the deploy with the template's error message rather than // timing out per subtest. -func deployFixture(t *testing.T, ctx context.Context, clients *e2e.Clients, bucket string) string { +func deployFixture(t *testing.T, ctx context.Context, clients *e2e.Clients) string { t.Helper() atespace, _ := e2e.DeploySubstrateFixture(t, ctx, clients, e2e.SubstrateFixtureManifests{ Pool: "internal/e2e/fixtures/capabilities/capabilities.yaml.tmpl", Template: "internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl", - }, bucket, "capabilities", nil) + }, "capabilities", nil) return atespace } diff --git a/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go b/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go index 3c37ea65e5..3149ca2925 100644 --- a/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go +++ b/internal/e2e/suites/combinedvolumes/combinedvolumes_test.go @@ -182,13 +182,8 @@ func storageClassOrEmpty(ctx context.Context, t *testing.T, clients *e2e.Clients func createTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, ns *e2e.Namespace, fixtureImage, storageClass string) *ateapi.ActorTemplate { t.Helper() - env, err := e2e.CheckEnv("BUCKET_NAME") - if err != nil { - t.Fatalf("CheckEnv: %v", err) - } - // The probe supplies this suite's container image and resolved runtime. - probeAtespace, _ := e2e.DeployProbe(t, env["BUCKET_NAME"], "combinedvolumes") + probeAtespace, _ := e2e.DeployProbe(t, "combinedvolumes") src := e2e.SubstrateFixture{ Atespace: probeAtespace, Name: probeName, @@ -206,7 +201,7 @@ func createTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, ns // scheduler cannot hand its workers to another suite's actors. Labels: map[string]string{"combinedvolumes": ns.Name}, SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: fmt.Sprintf("gs://%s/%s/", env["BUCKET_NAME"], ns.Name), + StorageLocation: e2e.SnapshotLocation(t, ns.Name), }, Modify: func(tmpl *ateapi.ActorTemplate) { // Every volume is mounted at two paths, covering the read-only diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 0a30906e50..11d87efafa 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -1338,11 +1338,6 @@ func deletePausedActor(ctx context.Context, t *testing.T, clients *e2e.Clients, } func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, base string, fidelity ateapi.SnapshotFidelity, modifyTemplate func(*ateapi.ActorTemplate)) (*ateapi.ActorTemplate, error) { - env, err := e2e.CheckEnv("BUCKET_NAME") - if err != nil { - t.Fatalf("CheckEnv failed: %v", err) - } - // The whole suite shares demoAtespace, so the per-test suffix keeps // template names unique. name := base + "-" + nsObj.Name @@ -1351,17 +1346,24 @@ func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e Name: name, PoolName: base, // One actor at a time, plus a spare for the worker-deletion tests. - PoolReplicas: 2, - Labels: map[string]string{"demo": nsObj.Name}, - SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: "gs://" + env["BUCKET_NAME"] + "/ate-demo-" + name, - PreferredFidelity: fidelity, - }, - Modify: modifyTemplate, + PoolReplicas: 2, + Labels: map[string]string{"demo": nsObj.Name}, + SnapshotConfig: templateSnapshotConfig(t, fidelity, "ate-demo-"+name), + Modify: modifyTemplate, }) return at, nil } +// templateSnapshotConfig returns a SnapshotConfig that captures at fidelity, +// stored under e2e.SnapshotLocation(prefix). +func templateSnapshotConfig(t *testing.T, fidelity ateapi.SnapshotFidelity, prefix string) *ateapi.SnapshotConfig { + t.Helper() + return &ateapi.SnapshotConfig{ + PreferredFidelity: fidelity, + StorageLocation: e2e.SnapshotLocation(t, prefix), + } +} + func createActorTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, fidelity ateapi.SnapshotFidelity) (*ateapi.ActorTemplate, error) { return createActorTemplateInternal(ctx, t, clients, nsObj, "counter", fidelity, nil) } diff --git a/internal/e2e/suites/demo/golden_test.go b/internal/e2e/suites/demo/golden_test.go index c8a7c0acfe..4ff9e3deae 100644 --- a/internal/e2e/suites/demo/golden_test.go +++ b/internal/e2e/suites/demo/golden_test.go @@ -35,10 +35,6 @@ func TestGoldenSnapshotLifecycle(t *testing.T) { ctx := t.Context() clients := e2e.GetClients() ns := e2e.CreateNamespace(t) - env, err := e2e.CheckEnv("BUCKET_NAME") - if err != nil { - t.Fatal(err) - } // A distinct atespace catches placement in a system atespace and // keeps cleanup independent of the suite's shared demo atespace. atespace := "golden-" + ns.Name @@ -49,11 +45,8 @@ func TestGoldenSnapshotLifecycle(t *testing.T) { }) created := e2e.CreateSubstrateCounterTemplate(ctx, t, clients, ns.Name, e2e.SubstrateTemplateOptions{ Atespace: atespace, Name: "counter", PoolName: "counter", PoolReplicas: 1, - Labels: map[string]string{"golden-e2e": ns.Name}, - SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: "gs://" + env["BUCKET_NAME"] + "/" + atespace, - PreferredFidelity: fidelity, - }, + Labels: map[string]string{"golden-e2e": ns.Name}, + SnapshotConfig: templateSnapshotConfig(t, fidelity, atespace), }) templateRef := e2e.TemplateRef(created) diff --git a/internal/e2e/suites/demo/updatetemplate_test.go b/internal/e2e/suites/demo/updatetemplate_test.go index a39d404e9f..a0ae526a70 100644 --- a/internal/e2e/suites/demo/updatetemplate_test.go +++ b/internal/e2e/suites/demo/updatetemplate_test.go @@ -64,18 +64,13 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapi.SnapshotFidelity) { ctx := context.Background() clients := e2e.GetClients() - env, err := e2e.CheckEnv("BUCKET_NAME") - if err != nil { - t.Fatalf("CheckEnv failed: %v", err) - } - // Template A: the plain counter. Template B: the same workload, but its // command additionally validates that the file template A's sprint left // in the durable dir is readable — the response's "file content" both // proves B's spec took effect and re-reads the preserved data. nameA, nameB := "update-a-"+nsObj.Name, "update-b-"+nsObj.Name - createdA := createUpdateTestTemplate(ctx, t, clients, nsObj, nameA, "update-a", env["BUCKET_NAME"], fidelity, nil) - createdB := createUpdateTestTemplate(ctx, t, clients, nsObj, nameB, "update-b", env["BUCKET_NAME"], fidelity, func(tmpl *ateapi.ActorTemplate) { + createdA := createUpdateTestTemplate(ctx, t, clients, nsObj, nameA, "update-a", fidelity, nil) + createdB := createUpdateTestTemplate(ctx, t, clients, nsObj, nameB, "update-b", fidelity, func(tmpl *ateapi.ActorTemplate) { ctr := tmpl.Containers[0] ctr.Command = append(ctr.Command, "--validate-existing-file-path=/home/counter/a.txt") }) @@ -322,19 +317,18 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapi.SnapshotFidelity) { // createUpdateTestTemplate creates a per-test WorkerPool plus a substrate // ActorTemplate copying the deployed counter fixture's resolved runtime, -// capturing at the scope under test on suspend. -func createUpdateTestTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, name, poolName, bucket string, fidelity ateapi.SnapshotFidelity, modify func(*ateapi.ActorTemplate)) *ateapi.ActorTemplate { +// capturing at the scope under test on suspend. Both templates of a test share +// one snapshot location, which UpdateActor requires once the actor has a +// snapshot. +func createUpdateTestTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, name, poolName string, fidelity ateapi.SnapshotFidelity, modify func(*ateapi.ActorTemplate)) *ateapi.ActorTemplate { t.Helper() return e2e.CreateSubstrateCounterTemplate(ctx, t, clients, nsObj.Name, e2e.SubstrateTemplateOptions{ - Atespace: demoAtespace, - Name: name, - PoolName: poolName, - PoolReplicas: 2, - Labels: map[string]string{"demo": nsObj.Name}, - SnapshotConfig: &ateapi.SnapshotConfig{ - StorageLocation: "gs://" + bucket + "/ate-demo-" + nsObj.Name, - PreferredFidelity: fidelity, - }, - Modify: modify, + Atespace: demoAtespace, + Name: name, + PoolName: poolName, + PoolReplicas: 2, + Labels: map[string]string{"demo": nsObj.Name}, + SnapshotConfig: templateSnapshotConfig(t, fidelity, "ate-demo-"+nsObj.Name), + Modify: modify, }) } diff --git a/internal/e2e/suites/egresscredinject/egresscredinject_test.go b/internal/e2e/suites/egresscredinject/egresscredinject_test.go index ae64812a12..f4575f7bf5 100644 --- a/internal/e2e/suites/egresscredinject/egresscredinject_test.go +++ b/internal/e2e/suites/egresscredinject/egresscredinject_test.go @@ -111,8 +111,7 @@ var withPlaceholder = []string{ // hack/setup-e2e-egress-tls-kind.sh // hack/run-e2e-kind.sh ./internal/e2e/suites/egresscredinject -v -args --no-color func TestActorEgressCredentialInjection(t *testing.T) { - env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") - if err != nil { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { t.Fatalf("CheckEnv failed: %v", err) } ctx := context.Background() @@ -124,7 +123,7 @@ func TestActorEgressCredentialInjection(t *testing.T) { e2e.ConfigureCredentialProvider(t) - probeNamespace, _ = e2e.DeployProbe(t, env["BUCKET_NAME"], "egresscredinject", e2e.WithTrustBundle()) + probeNamespace, _ = e2e.DeployProbe(t, "egresscredinject", e2e.WithTrustBundle()) const id = "probe-credinject" createAndResumeActor(t, ctx, clients, id, echoHost) diff --git a/internal/e2e/suites/egressmitm/egressmitm_test.go b/internal/e2e/suites/egressmitm/egressmitm_test.go index 2d6da343b3..47d6b32483 100644 --- a/internal/e2e/suites/egressmitm/egressmitm_test.go +++ b/internal/e2e/suites/egressmitm/egressmitm_test.go @@ -67,8 +67,7 @@ var probeNamespace string // The micro-VM variant additionally needs the micro-VM deps installed // (hack/run-microvm-demo-kind.sh, or hack/install-microvm-deps.sh --install). func TestActorEgressMITMTrust(t *testing.T) { - env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") - if err != nil { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { t.Fatalf("CheckEnv failed: %v", err) } ctx := context.Background() @@ -92,7 +91,7 @@ func TestActorEgressMITMTrust(t *testing.T) { passthrough := e2e.DeployServerPod(t, ctx, passthroughSpec) originRoots := "ca=" + url.QueryEscape(e2e.ServiceDNSTrustBundle(t, ctx, clients)) - probeNamespace, _ = e2e.DeployProbe(t, env["BUCKET_NAME"], "egressmitm", e2e.WithTrustBundle(), e2e.WithSystemRoots()) + probeNamespace, _ = e2e.DeployProbe(t, "egressmitm", e2e.WithTrustBundle(), e2e.WithSystemRoots()) const id = "probe-mitm" createAndResumeActor(t, ctx, clients, id, mitm, passthrough) diff --git a/internal/e2e/suites/egressmitm/goldenboot_test.go b/internal/e2e/suites/egressmitm/goldenboot_test.go index 09215bc123..800ee080e2 100644 --- a/internal/e2e/suites/egressmitm/goldenboot_test.go +++ b/internal/e2e/suites/egressmitm/goldenboot_test.go @@ -45,8 +45,7 @@ var bootFetchManifests = e2e.SubstrateFixtureManifests{ // hack/run-e2e-kind.sh ./internal/e2e/suites/egressmitm -run TestGoldenBootEgress -v -args --no-color --preview=GoldenEgressPolicy func TestGoldenBootEgress(t *testing.T) { e2e.RequirePreview(t, preview.GateGoldenEgressPolicy) - env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") - if err != nil { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { t.Fatalf("CheckEnv failed: %v", err) } ctx := context.Background() @@ -56,7 +55,7 @@ func TestGoldenBootEgress(t *testing.T) { // DeploySubstrateFixture returns once every template has its golden // snapshot, which is the assertion. - atespace, _ := e2e.DeploySubstrateFixture(t, ctx, clients, bootFetchManifests, env["BUCKET_NAME"], "egressmitm-bootfetch", []string{e2e.EgressTrustBundleName}, + atespace, _ := e2e.DeploySubstrateFixture(t, ctx, clients, bootFetchManifests, "egressmitm-bootfetch", []string{e2e.EgressTrustBundleName}, e2e.WithTemplateValue("BOOT_FETCH_HOST", origin.SvcName()), e2e.WithTemplateValue("BOOT_FETCH_URL", "https://"+origin.SvcName()+"/healthz")) diff --git a/internal/e2e/suites/identity/identity_test.go b/internal/e2e/suites/identity/identity_test.go index 60d3bbda0e..e68a18fec7 100644 --- a/internal/e2e/suites/identity/identity_test.go +++ b/internal/e2e/suites/identity/identity_test.go @@ -71,8 +71,7 @@ type whoamiResponse struct { // (e.g. a timestamped-directory symlink swap) would break every held fd of // any actor that ever touched a system-info file. func TestActorIdentity_AfterRestore_IsOwnID_NotGolden(t *testing.T) { - env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") - if err != nil { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { t.Fatalf("CheckEnv failed: %v", err) } ctx := context.Background() @@ -83,7 +82,7 @@ func TestActorIdentity_AfterRestore_IsOwnID_NotGolden(t *testing.T) { // contents. wantTrust := e2e.AddEgressCA(t, ctx, clients, "ate-e2e-probe-trust") var tmpl *ateapi.ActorTemplate - probeNamespace, tmpl = e2e.DeployProbe(t, env["BUCKET_NAME"], "identity", e2e.WithTrustBundle()) + probeNamespace, tmpl = e2e.DeployProbe(t, "identity", e2e.WithTrustBundle()) // The golden actor's id, for the not-golden assertion below. Coupled to // the reconciler's naming: the golden actor is named after the template's // UID (cmd/ateapi template reconciler), so a naming change there weakens diff --git a/internal/e2e/suites/multiactor/multiactor_test.go b/internal/e2e/suites/multiactor/multiactor_test.go index f496c557b3..bb43cffa7c 100644 --- a/internal/e2e/suites/multiactor/multiactor_test.go +++ b/internal/e2e/suites/multiactor/multiactor_test.go @@ -44,15 +44,14 @@ type whoamiResponse struct { func TestTwoActorsShareOneWorker(t *testing.T) { ctx := context.Background() clients := e2e.GetClients() - env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") - if err != nil { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { t.Fatal(err) } e2e.DeploySubstrateFixture(t, ctx, clients, e2e.SubstrateFixtureManifests{ Pool: "internal/e2e/fixtures/probe/probe-multiactor.yaml.tmpl", Template: "internal/e2e/fixtures/probe/probe-multiactor-template.yaml.tmpl", - }, env["BUCKET_NAME"], "multiactor", nil) + }, "multiactor", nil) names := []string{"ma-first", "ma-second"} pods := map[string]string{} diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index b034e457cd..97a5a2f3fb 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -137,13 +137,12 @@ func TestIngressProtocolDowngrade(t *testing.T) { // deploys as a plain pod, so the two directions cannot disagree about what a // working RPC is. func TestIngressGRPC(t *testing.T) { - env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") - if err != nil { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { t.Fatalf("CheckEnv failed: %v", err) } ctx := context.Background() - fixture := deployGRPCEchoTemplate(t, ctx, env["BUCKET_NAME"]) + fixture := deployGRPCEchoTemplate(t, ctx) _, actorName, _ := createAndResumeSubstrateActor(t, ctx, "grpcingress", fixture) actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} ctx = metadata.AppendToOutgoingContext(ctx, @@ -262,9 +261,9 @@ func TestIngressGRPC(t *testing.T) { // under test, waits for its golden snapshot and returns it. The suite gets its // own copy of the fixture: suite packages run as concurrent processes, so a // shared one would be deleted out from under another. -func deployGRPCEchoTemplate(t *testing.T, ctx context.Context, bucket string) e2e.SubstrateFixture { +func deployGRPCEchoTemplate(t *testing.T, ctx context.Context) e2e.SubstrateFixture { t.Helper() - atespace, _ := e2e.DeploySubstrateFixture(t, ctx, e2e.GetClients(), grpcEchoFixtureManifests, bucket, "networking", nil) + atespace, _ := e2e.DeploySubstrateFixture(t, ctx, e2e.GetClients(), grpcEchoFixtureManifests, "networking", nil) return e2e.SubstrateFixture{ Atespace: atespace, Name: "grpcecho", diff --git a/internal/e2e/suites/networking/websocketingress_test.go b/internal/e2e/suites/networking/websocketingress_test.go index 5a497badf5..104b3eddee 100644 --- a/internal/e2e/suites/networking/websocketingress_test.go +++ b/internal/e2e/suites/networking/websocketingress_test.go @@ -32,15 +32,14 @@ import ( // under test and waits for its golden snapshot. func deployWebsocketFixture(t *testing.T, ctx context.Context) e2e.SubstrateFixture { t.Helper() - env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") - if err != nil { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { t.Fatalf("CheckEnv failed: %v", err) } atespace, _ := e2e.DeploySubstrateFixture(t, ctx, e2e.GetClients(), e2e.SubstrateFixtureManifests{ Pool: "internal/e2e/fixtures/testserver/websocket.yaml.tmpl", Template: "internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl", - }, env["BUCKET_NAME"], "websocket", nil) + }, "websocket", nil) return e2e.SubstrateFixture{ Atespace: atespace, diff --git a/internal/e2e/suites/parking/parking_test.go b/internal/e2e/suites/parking/parking_test.go index 76b1d7b1fa..42df00756c 100644 --- a/internal/e2e/suites/parking/parking_test.go +++ b/internal/e2e/suites/parking/parking_test.go @@ -199,14 +199,10 @@ func TestRequestParking(t *testing.T) { // actors (see probe-parking.yaml.tmpl). func deployParkingFixture(t *testing.T, ctx context.Context, clients *e2e.Clients) *ateapi.ActorTemplate { t.Helper() - env, err := e2e.CheckEnv("BUCKET_NAME") - if err != nil { - t.Fatalf("CheckEnv failed: %v", err) - } _, templates := e2e.DeploySubstrateFixture(t, ctx, clients, e2e.SubstrateFixtureManifests{ Pool: "internal/e2e/fixtures/probe/probe-parking.yaml.tmpl", Template: "internal/e2e/fixtures/probe/probe-parking-template.yaml.tmpl", - }, env["BUCKET_NAME"], "parking", nil) + }, "parking", nil) return templates[0] } diff --git a/internal/e2e/suites/security/crossatespace_test.go b/internal/e2e/suites/security/crossatespace_test.go index 1597fadcaf..ebd2b4b885 100644 --- a/internal/e2e/suites/security/crossatespace_test.go +++ b/internal/e2e/suites/security/crossatespace_test.go @@ -95,14 +95,13 @@ type atespaces struct { // atespace's golden snapshot, which on the micro-VM lane is a cold boot plus a // checkpoint, so per-subtest fixtures would dominate the runtime. func TestCrossAtespaceIsolation(t *testing.T) { - env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") - if err != nil { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { t.Fatalf("CheckEnv failed: %v", err) } ctx := context.Background() clients := e2e.GetClients() - ns := deployAtespaces(t, ctx, clients, env["BUCKET_NAME"]) + ns := deployAtespaces(t, ctx, clients) rc, err := e2e.NewRouterClient(t, ctx) if err != nil { @@ -306,16 +305,16 @@ func testTagScopeBoundary(t *testing.T, ctx context.Context, clients *e2e.Client // deployAtespaces installs both atespaces and returns their names. A goes // first: it brings the only WorkerPool, and B's golden snapshot needs a worker // to boot on. -func deployAtespaces(t *testing.T, ctx context.Context, clients *e2e.Clients, bucket string) atespaces { +func deployAtespaces(t *testing.T, ctx context.Context, clients *e2e.Clients) atespaces { t.Helper() a, _ := e2e.DeploySubstrateFixture(t, ctx, clients, e2e.SubstrateFixtureManifests{ Pool: "internal/e2e/fixtures/security/atespace-a-pool.yaml.tmpl", Template: "internal/e2e/fixtures/security/atespace-template.yaml.tmpl", - }, bucket, "sec-a", nil) + }, "sec-a", nil) b, _ := e2e.DeploySubstrateFixture(t, ctx, clients, e2e.SubstrateFixtureManifests{ Pool: "internal/e2e/fixtures/security/atespace-b-namespace.yaml.tmpl", Template: "internal/e2e/fixtures/security/atespace-template.yaml.tmpl", - }, bucket, "sec-b", nil) + }, "sec-b", nil) if a == b { t.Fatalf("both fixtures rendered to atespace %q — the fixture no longer isolates anything", a) } diff --git a/internal/e2e/suites/sizing/sizing_test.go b/internal/e2e/suites/sizing/sizing_test.go index f853cd6d20..3d54e8c96a 100644 --- a/internal/e2e/suites/sizing/sizing_test.go +++ b/internal/e2e/suites/sizing/sizing_test.go @@ -62,14 +62,13 @@ type resourcesResponse struct { // the CPU quota and the cgroup memory limit, while the micro-VM sizes the guest // VM itself (see internal/sizing). func TestActorSizing_SandboxObservesDeclaredLimits(t *testing.T) { - env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") - if err != nil { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { t.Fatalf("CheckEnv failed: %v", err) } ctx := context.Background() clients := e2e.GetClients() - deploySizedProbe(t, ctx, clients, env["BUCKET_NAME"]) + deploySizedProbe(t, ctx, clients) const id = "sized-actor" createAndResumeActor(t, ctx, clients, id) @@ -108,12 +107,12 @@ func TestActorSizing_SandboxObservesDeclaredLimits(t *testing.T) { // deploySizedProbe installs the sized probe fixture for the sandbox class // under test and waits for its golden snapshot. -func deploySizedProbe(t *testing.T, ctx context.Context, clients *e2e.Clients, bucket string) { +func deploySizedProbe(t *testing.T, ctx context.Context, clients *e2e.Clients) { t.Helper() e2e.DeploySubstrateFixture(t, ctx, clients, e2e.SubstrateFixtureManifests{ Pool: "internal/e2e/fixtures/probe/probe-sized.yaml.tmpl", Template: "internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl", - }, bucket, "sizing", nil) + }, "sizing", nil) } func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Clients, id string) { diff --git a/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go b/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go index 0b9af6acf8..8460816284 100644 --- a/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go +++ b/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go @@ -278,16 +278,12 @@ func suspendSync(t *testing.T, ctx context.Context, clients *e2e.Clients, atespa // their own snapshots, it reads its own and not the sibling's. Tokens the // exchange must refuse are checked alongside. func TestExchangeScopesTheTokenToTheActor(t *testing.T) { - env, err := e2e.CheckEnv("BUCKET_NAME") - if err != nil { - t.Fatalf("CheckEnv failed: %v", err) - } ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) defer cancel() clients := e2e.GetClients() requireGCS(t, ctx, clients) - atespace, _ := e2e.DeployProbe(t, env["BUCKET_NAME"], "snapcreds") + atespace, _ := e2e.DeployProbe(t, "snapcreds") sign := newSigner(t, ctx, clients) xchg := dialExchange(t, ctx, clients) stamp := time.Now().UTC().Format("20060102T150405Z") @@ -415,14 +411,14 @@ func TestRepointMovesTheGrant(t *testing.T) { clients := e2e.GetClients() requireGCS(t, ctx, clients) - atespace, probe := e2e.DeployProbe(t, env["BUCKET_NAME"], "snapcreds-repoint") + atespace, probe := e2e.DeployProbe(t, "snapcreds-repoint") sign := newSigner(t, ctx, clients) xchg := dialExchange(t, ctx, clients) stamp := time.Now().UTC().Format("20060102T150405Z") - // Template B is the probe template under another location in the same - // bucket. Its golden is not what the actor resumes from, but the actor's - // next snapshot goes under B's location. + // Template B is the probe template under another location in + // $BUCKET_NAME. B's golden is not what the actor resumes from, but the + // actor's next snapshot goes under B's location. const nameB = "probe-repoint" locationB := "gs://" + env["BUCKET_NAME"] + "/ate-e2e-repoint-" + stamp + "/" templateB := proto.Clone(probe).(*ateapi.ActorTemplate) diff --git a/internal/e2e/template.go b/internal/e2e/template.go index 019cac8762..72355b4568 100644 --- a/internal/e2e/template.go +++ b/internal/e2e/template.go @@ -23,6 +23,7 @@ import ( atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -45,7 +46,8 @@ type SubstrateTemplateOptions struct { // Labels tie the template's workerSelector to the pool, keeping this // pool's workers invisible to other namespaces' actors. Labels map[string]string - // SnapshotConfig for the new template; nil copies the source's. + // SnapshotConfig for the new template. nil copies the source's, with its + // own SnapshotLocation. SnapshotConfig *ateapi.SnapshotConfig // Modify, when set, edits the template before it is created. Modify func(*ateapi.ActorTemplate) @@ -101,7 +103,11 @@ func CreateSubstrateTemplateFrom(ctx context.Context, t *testing.T, clients *Cli snapshots := opts.SnapshotConfig if snapshots == nil { - snapshots = srcTmpl.GetSnapshotConfig() + snapshots = &ateapi.SnapshotConfig{} + if src := srcTmpl.GetSnapshotConfig(); src != nil { + snapshots = proto.Clone(src).(*ateapi.SnapshotConfig) + } + snapshots.StorageLocation = SnapshotLocation(t, "ate-e2e-"+opts.Atespace+"-"+opts.Name) } tmpl := &ateapi.ActorTemplate{ Metadata: &ateapi.ResourceMetadata{Atespace: opts.Atespace, Name: opts.Name}, diff --git a/internal/preview/preview.go b/internal/preview/preview.go index 8442ff875e..0d3d17c245 100644 --- a/internal/preview/preview.go +++ b/internal/preview/preview.go @@ -45,9 +45,10 @@ const ( // GateGoldenEgressPolicy controls configuring a golden actor's egress policy on its template. GateGoldenEgressPolicy Gate = "GoldenEgressPolicy" - // GatePerTemplateSnapshotLocation controls whether an ActorTemplate's - // snapshotConfig.storage_location is honored. When disabled, snapshots - // are stored under ate-api-server's --snapshot-location. + // GatePerTemplateSnapshotLocation controls whether an ActorTemplate may + // set snapshotConfig.storage_location. When disabled, the API rejects the + // field and snapshots are stored under ate-api-server's + // --snapshot-location. GatePerTemplateSnapshotLocation Gate = "PerTemplateSnapshotLocation" ) From eaf0e85c7622d021d833c13e8f8911e2854f9794 Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Thu, 29 Oct 2026 00:56:38 +0000 Subject: [PATCH 47/53] ate-api-server's --egress-gateway-spiffe-id is repeatable, and ate-setup sets it from ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS (#3360) Why is this needed? Credential plugins may need an actor JWT to exchange for other credentials, e.g. through an RFC 8693 exchange. Today only `egress_gateway` can call `MintActorJwt`; ate-api-server grants that role to one SPIFFE ID. It's not possible to grant permissions to generate Actor JWTs to any other identity. This makes `--egress-gateway-spiffe-id` repeatable and grants `egress_gateway` to every ID passed. An operator who deploys a plugin lists its SPIFFE ID in ate-setup's new `ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS` setting (`--egress-gateway-extra-spiffe-ids`). ate-setup then passes `atenet-egress`'s ID first, since setting the flag replaces the default, followed by the listed IDs. The default is still `atenet-egress` alone, and an empty ID still fails startup. There is no API or manifest change. A listed ID gets the whole `egress_gateway` role: besides minting actor JWTs, it can get and list actors, read their egress policies and mint actor certificates. Part of #1661. AI use: AI-assisted draft using Claude. - [ ] Tests pass - [x] Docs updated --------- Signed-off-by: Max Thompson --- cmd/ate-setup/commands.md | 1 + cmd/ate-setup/internal/config/config.go | 8 +++ cmd/ate-setup/internal/config/config_test.go | 39 +++++++++++++ .../internal/config/coverage_test.go | 2 +- cmd/ate-setup/internal/config/load.go | 13 ++++- cmd/ate-setup/internal/config/setting.go | 7 +++ cmd/ate-setup/internal/steps/apienv.go | 46 +++++++++++++-- cmd/ate-setup/internal/steps/apienv_test.go | 57 +++++++++++++++++++ cmd/ateapi/internal/authz/authz_util.go | 14 +++-- cmd/ateapi/internal/authz/authz_util_test.go | 14 +++-- cmd/ateapi/internal/authz/interceptor_test.go | 2 +- .../internal/store/atepg/common_test.go | 2 +- .../internal/store/storetest/storetest.go | 2 +- cmd/ateapi/main.go | 4 +- docs/authorization.md | 2 +- docs/operator-install.md | 1 + 16 files changed, 189 insertions(+), 25 deletions(-) diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 87672d5217..8d090d24d6 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -27,6 +27,7 @@ every setting with its environment variable. | `--cordon-system-infra-components` | Keep the system infra components, such as ate-api-server, atenet-router, atenet-egress and postgres, off the worker nodes. Assumes a small shared pool labeled and tainted `ate.dev/workloadType=ate-system-infra:NoSchedule`, across which each workload's replicas are spread, and a one-node pool labeled and tainted `ate.dev/workloadType=ate-postgres:NoSchedule` for postgres alone. `ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS=true` when the flag is absent | | `--fake-data-plane` | Install the fake data plane for control-plane benchmarks: fake-atelet takes atelet's place in its DaemonSet, taking its objectstore-plugin's storage settings, ate-controller is scaled to zero, and fake-workersync honors every WorkerPool with fake Workers on the fake-atelet nodes. Reaches `deploy ate-system`, `deploy atelet`, and `deploy ate-controller`; only `deploy ate-system` switches a cluster between the real and fake data planes, and the other two refuse a data plane other than the installed one. `deploy ate-system --fake-data-plane` refuses a cluster whose WorkerPools have worker pods. fake-atelet runs with its defaults; a benchmark tunes it through the optional `fake-atelet-tuning` ConfigMap, whose `FAKE_ATELET_*` variables set its flags, and which no install creates or resets. Actors do not exist; never use it on a cluster that serves real actors. `ATE_INSTALL_FAKE_DATA_PLANE=true` when the flag is absent | | `--ingress-allowed-spiffe-ids IDS` | Comma-separated SPIFFE IDs (exact match) of the clients the ingress router lets reach actors; any listed client may reach any actor. Unset, no client may reach actors through the router. Both dataplanes require a Pod Identity CA client certificate on HTTPS and TLS CONNECT. The SPIFFE URI SAN allowlist may be checked during or after TLS, including after CONNECT acceptance, but unlisted clients must be denied before actor routing. The podidentity signer issues `spiffe://cluster.local/ns//sa/`. `ATE_INGRESS_ALLOWED_SPIFFE_IDS` when the flag is absent | +| `--egress-gateway-extra-spiffe-ids IDS` | Comma-separated SPIFFE IDs that hold ate-api-server's `egress_gateway` role next to atenet-egress, such as a credential provider that mints actor JWTs. ate-setup passes atenet-egress's ID and then each listed ID as repeated `--egress-gateway-spiffe-id` flags; unset passes none, leaving ate-api-server's default. `ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS` when the flag is absent | | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | | `--credential-provider JSON` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/egress-credential-injection.md`](../../docs/egress-credential-injection.md). `ATE_CREDENTIAL_PROVIDER` when the flag is absent | | `--ateapi-log-level LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). The server picks up a change without restarting. `ATE_API_LOG_LEVEL` when the flag is absent | diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 2706b81cad..a44fdf816a 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -121,6 +121,10 @@ type Config struct { // owner on global:root (ATE_API_AUTHZ_BOOTSTRAP_OWNERS). APIAuthzBootstrapOwners string + // EgressGatewayExtraSPIFFEIDs hold ate-api-server's egress_gateway role + // next to atenet-egress (ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS). + EgressGatewayExtraSPIFFEIDs []string + // ActorJWTAlgorithm is the signing algorithm of the key in a new actor JWT // pool (ACTOR_JWT_ALGORITHM): ES256 or RS256. ActorJWTAlgorithm string @@ -521,6 +525,10 @@ func (c *Config) ScriptEnv() []string { if len(c.IngressAllowedSPIFFEIDs) > 0 { merged["ATE_INGRESS_ALLOWED_SPIFFE_IDS"] = strings.Join(c.IngressAllowedSPIFFEIDs, ",") } + delete(merged, "ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS") + if len(c.EgressGatewayExtraSPIFFEIDs) > 0 { + merged["ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS"] = strings.Join(c.EgressGatewayExtraSPIFFEIDs, ",") + } if c.AdditionalEgressExtprocService != "" { merged["ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE"] = c.AdditionalEgressExtprocService } diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 074fa5f99e..5429ece623 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -61,6 +61,7 @@ func loadEnv(t *testing.T) { "ATE_API_POSTGRES_SERVER_CA_FILE", "ATE_ATENET_DATAPLANE", "ATE_CREDENTIAL_PROVIDER", + "ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS", "ATE_IMAGE_REPO", "ATE_IMAGE_TAG", "ATE_INGRESS_ALLOWED_SPIFFE_IDS", @@ -316,6 +317,44 @@ func TestLoadIngressAllowedSPIFFEIDs(t *testing.T) { } } +func TestLoadEgressGatewayExtraSPIFFEIDs(t *testing.T) { + const a, b = "spiffe://cluster.local/ns/a/sa/a", "spiffe://cluster.local/ns/b/sa/b" + for _, tc := range []struct { + name string + env string + want []string + wantErr string + }{ + {name: "unset", want: nil}, + {name: "several, with stray separators", env: " " + a + ",," + b + ", ", want: []string{a, b}}, + {name: "not a SPIFFE ID", env: a + ",https://example.com/x", wantErr: "ateapi.authz.egressGatewayExtraSPIFFEIDs must be a comma-separated list of SPIFFE IDs"}, + } { + t.Run(tc.name, func(t *testing.T) { + loadEnv(t) + if tc.env != "" { + t.Setenv("ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS", tc.env) + } + cfg, err := Load(Options{}) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("Load() error = %v, want one containing %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if !slices.Equal(cfg.EgressGatewayExtraSPIFFEIDs, tc.want) { + t.Errorf("EgressGatewayExtraSPIFFEIDs = %q, want %q", cfg.EgressGatewayExtraSPIFFEIDs, tc.want) + } + got, exported := scriptEnvMap(t, cfg)["ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS"] + if want := strings.Join(tc.want, ","); exported != (want != "") || got != want { + t.Errorf("ScriptEnv() ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS = %q (exported %v), want %q", got, exported, want) + } + }) + } +} + // The credential exchange impersonates the transfer account only on GCS, so // only a GKE deploy needs one, and its email needs both the name and the // project. diff --git a/cmd/ate-setup/internal/config/coverage_test.go b/cmd/ate-setup/internal/config/coverage_test.go index 8f23e55ea5..5585769f7a 100644 --- a/cmd/ate-setup/internal/config/coverage_test.go +++ b/cmd/ate-setup/internal/config/coverage_test.go @@ -90,7 +90,7 @@ func valueFor(key string) string { return RouterEnvoy case "clusterSize": return ClusterSizeSize0 - case "ingressAllowedSPIFFEIDs": + case "ingressAllowedSPIFFEIDs", "ateapi.authz.egressGatewayExtraSPIFFEIDs": return "spiffe://cluster.local/ns/a/sa/b" case "ateapi.postgres.cloudsql.ipType": return CloudSQLIPTypePSC diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index c553243a67..056e4e815b 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -223,7 +223,8 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro ClusterLocation: r.String("gcp.clusterLocation"), ExpectedJWTIssuer: r.String("ateapi.expectedJWTIssuer"), - APIAuthzBootstrapOwners: bootstrapOwners, + APIAuthzBootstrapOwners: bootstrapOwners, + EgressGatewayExtraSPIFFEIDs: splitList(r.String("ateapi.authz.egressGatewayExtraSPIFFEIDs")), BucketName: r.String("storage.bucketName"), SnapshotLocation: r.String("storage.snapshotLocation"), @@ -406,14 +407,20 @@ func validateResolved(cfg *Config, r *Resolved) error { return &InvalidError{Value: size, Want: ClusterSizeSize0 + " or " + ClusterSizeSize10} } - // The router rejects a malformed ID at startup; catch it before the - // rollout does. + // The router and ate-api-server reject a malformed ID at startup; catch it + // before the rollout does. for _, id := range cfg.IngressAllowedSPIFFEIDs { if _, err := spiffeid.FromString(id); err != nil { ids, _ := r.Value("ingressAllowedSPIFFEIDs") return &InvalidError{Value: ids, Want: fmt.Sprintf("a comma-separated list of SPIFFE IDs (%q: %v)", id, err)} } } + for _, id := range cfg.EgressGatewayExtraSPIFFEIDs { + if _, err := spiffeid.FromString(id); err != nil { + ids, _ := r.Value("ateapi.authz.egressGatewayExtraSPIFFEIDs") + return &InvalidError{Value: ids, Want: fmt.Sprintf("a comma-separated list of SPIFFE IDs (%q: %v)", id, err)} + } + } switch cfg.APILogLevel { case "", "debug", "info", "warn", "error": diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 0c16e1b897..6fe546a358 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -187,6 +187,13 @@ var Registry = []Setting{ Usage: "Comma-separated principals that always hold owner on global:root. " + "Unset means kubectl-ate's ServiceAccount, system:serviceaccount::ate-client", }, + { + Key: "ateapi.authz.egressGatewayExtraSPIFFEIDs", Env: "ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS", + Flag: "egress-gateway-extra-spiffe-ids", Kind: KindString, + Usage: "Comma-separated SPIFFE IDs that hold ate-api-server's egress_gateway role next to atenet-egress, " + + "such as a credential provider that mints actor JWTs. The role also reads actors and mints actor " + + "certificates. The podidentity signer issues spiffe://cluster.local/ns//sa/", + }, { // The egress gateway's credential provider, as a JSON object. Upstream // takes it as one value on one flag; it is carried the same way here diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 755b346af0..acb50a3667 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -357,6 +357,9 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er if err := setAPIServerAuthzArgs(objs, e.Cfg.APIAuthzBootstrapOwners); err != nil { return err } + if err := setAPIServerEgressGatewayArgs(objs, e.Namespace(), e.Cfg.EgressGatewayExtraSPIFFEIDs); err != nil { + return err + } if msg := sharedBootstrapOwnerWarning(e.Cfg.APIAuthzBootstrapOwners, e.Namespace()); msg != "" { log.Warnf("%s", msg) } @@ -367,9 +370,40 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er const apiServerAuthzFlag = "--authz-bootstrap-owners" // setAPIServerAuthzArgs sets ate-api-server's bootstrap owners flag on its -// container in objs, replacing any the manifest already carries. A changed -// flag changes the pod template, so it rolls the Deployment. +// container in objs, replacing any the manifest already carries. func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, bootstrapOwners string) error { + var values []string + if strings.Trim(bootstrapOwners, " ,") != "" { + values = []string{bootstrapOwners} + } + return setAPIServerFlag(objs, apiServerAuthzFlag, values) +} + +// apiServerEgressGatewayFlag is the ate-api-server flag +// setAPIServerEgressGatewayArgs owns. +const apiServerEgressGatewayFlag = "--egress-gateway-spiffe-id" + +// setAPIServerEgressGatewayArgs passes atenet-egress's SPIFFE ID and then each +// of extras, without duplicates, to ate-api-server's egress gateway flag. With +// no extras it drops the flag, leaving ate-api-server's default. +func setAPIServerEgressGatewayArgs(objs []*unstructured.Unstructured, namespace string, extras []string) error { + var ids []string + if len(extras) > 0 { + // Passing the flag replaces ate-api-server's default. + ids = []string{installdefaults.EgressSPIFFEID(namespace)} + for _, id := range extras { + if !slices.Contains(ids, id) { + ids = append(ids, id) + } + } + } + return setAPIServerFlag(objs, apiServerEgressGatewayFlag, ids) +} + +// setAPIServerFlag replaces every flag arg on ate-api-server's container in +// objs with one flag=value arg per value, in order. A changed flag changes the +// pod template, so it rolls the Deployment. +func setAPIServerFlag(objs []*unstructured.Unstructured, flag string, values []string) error { var dep *unstructured.Unstructured for _, obj := range objs { if obj.GetKind() == "Deployment" && obj.GetName() == "ate-api-server" { @@ -378,7 +412,7 @@ func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, bootstrapOwners st } } if dep == nil { - return fmt.Errorf("the manifest has no deployment/ate-api-server to set authorization flags on") + return fmt.Errorf("the manifest has no deployment/ate-api-server to set %s on", flag) } containers, found, err := unstructured.NestedSlice(dep.Object, "spec", "template", "spec", "containers") if err != nil || !found { @@ -394,10 +428,10 @@ func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, bootstrapOwners st return fmt.Errorf("%s: %w", kube.Describe(dep), err) } args = slices.DeleteFunc(args, func(arg string) bool { - return arg == apiServerAuthzFlag || strings.HasPrefix(arg, apiServerAuthzFlag+"=") + return arg == flag || strings.HasPrefix(arg, flag+"=") }) - if strings.Trim(bootstrapOwners, " ,") != "" { - args = append(args, apiServerAuthzFlag+"="+bootstrapOwners) + for _, v := range values { + args = append(args, flag+"="+v) } if err := unstructured.SetNestedStringSlice(container, args, "args"); err != nil { return fmt.Errorf("%s: %w", kube.Describe(dep), err) diff --git a/cmd/ate-setup/internal/steps/apienv_test.go b/cmd/ate-setup/internal/steps/apienv_test.go index 3c97dea3dc..db6694a706 100644 --- a/cmd/ate-setup/internal/steps/apienv_test.go +++ b/cmd/ate-setup/internal/steps/apienv_test.go @@ -695,6 +695,63 @@ spec: } } +func TestSetAPIServerEgressGatewayArgs(t *testing.T) { + const manifest = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: ate-api-server + namespace: ate-system +spec: + template: + spec: + containers: + - name: ate-api-server + args: + - --config-file=/run/apiconfig.ate.dev/apiconfig.textproto + - --egress-gateway-spiffe-id=stale +` + const ( + egress = "spiffe://cluster.local/ns/ate-system/sa/atenet-egress" + a = "spiffe://cluster.local/ns/ate-system/sa/sts-credential-provider" + b = "spiffe://cluster.local/ns/plugins/sa/other" + ) + for _, tc := range []struct { + name string + extras []string + want []string + }{ + { + name: "gateway first, then extras without duplicates", + extras: []string{a, egress, b, a}, + want: []string{ + "--config-file=/run/apiconfig.ate.dev/apiconfig.textproto", + "--egress-gateway-spiffe-id=" + egress, + "--egress-gateway-spiffe-id=" + a, + "--egress-gateway-spiffe-id=" + b, + }, + }, + { + name: "no extras drops the flag", + want: []string{"--config-file=/run/apiconfig.ate.dev/apiconfig.textproto"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + objs, err := kube.DecodeManifestBytes([]byte(manifest)) + if err != nil { + t.Fatalf("DecodeManifestBytes() error = %v", err) + } + if err := setAPIServerEgressGatewayArgs(objs, "ate-system", tc.extras); err != nil { + t.Fatalf("setAPIServerEgressGatewayArgs() error = %v", err) + } + containers, _, _ := unstructured.NestedSlice(objs[0].Object, "spec", "template", "spec", "containers") + args, _, _ := unstructured.NestedStringSlice(containers[0].(map[string]any), "args") + if diff := cmp.Diff(tc.want, args); diff != "" { + t.Errorf("ate-api-server args (-want +got):\n%s", diff) + } + }) + } +} + func TestCreateAPIServerEnvVarsOmitsAuthz(t *testing.T) { e := &Env{Cfg: &config.Config{ TransferGSAName: testTransferGSAName, ProjectID: testProjectID, diff --git a/cmd/ateapi/internal/authz/authz_util.go b/cmd/ateapi/internal/authz/authz_util.go index a85729f4d7..1e1734d6cd 100644 --- a/cmd/ateapi/internal/authz/authz_util.go +++ b/cmd/ateapi/internal/authz/authz_util.go @@ -233,7 +233,7 @@ func EnsureStoreAndModel(ctx context.Context, pool *pgxpool.Pool, fgaServer *ser // every check, independent of any stored AccessPolicy. type SystemPrincipals struct { ControllerSPIFFEID string - EgressGatewaySPIFFEID string + EgressGatewaySPIFFEIDs []string IngressRouterSPIFFEID string AteletSPIFFEID string APIServerSidecarSPIFFEID string @@ -289,16 +289,22 @@ func parseBootstrapOwners(ids []string) (map[string]struct{}, error) { // parseSystemPrincipals validates system's SPIFFE IDs and maps each OpenFGA // user string to the system component roles it holds. func parseSystemPrincipals(system SystemPrincipals) (map[string][]string, error) { - bindings := []struct { + if len(system.EgressGatewaySPIFFEIDs) == 0 { + return nil, fmt.Errorf("no %s SPIFFE ID", RoleEgressGateway) + } + type binding struct { role string spiffeID string - }{ + } + bindings := []binding{ {role: RoleController, spiffeID: system.ControllerSPIFFEID}, - {role: RoleEgressGateway, spiffeID: system.EgressGatewaySPIFFEID}, {role: RoleIngressRouter, spiffeID: system.IngressRouterSPIFFEID}, {role: RoleAtelet, spiffeID: system.AteletSPIFFEID}, {role: RoleAPIServerSidecar, spiffeID: system.APIServerSidecarSPIFFEID}, } + for _, id := range system.EgressGatewaySPIFFEIDs { + bindings = append(bindings, binding{role: RoleEgressGateway, spiffeID: id}) + } out := make(map[string][]string, len(bindings)) for _, b := range bindings { if _, err := spiffeid.FromString(b.spiffeID); err != nil { diff --git a/cmd/ateapi/internal/authz/authz_util_test.go b/cmd/ateapi/internal/authz/authz_util_test.go index 77a13c26cf..da08a0887f 100644 --- a/cmd/ateapi/internal/authz/authz_util_test.go +++ b/cmd/ateapi/internal/authz/authz_util_test.go @@ -26,7 +26,7 @@ import ( func testSystemPrincipals() SystemPrincipals { return SystemPrincipals{ ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEIDs: []string{installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace)}, IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), @@ -96,7 +96,7 @@ func TestParseSystemPrincipals(t *testing.T) { const router = "spiffe://cluster.local/ns/ate-system/sa/atenet-router" got, err := parseSystemPrincipals(SystemPrincipals{ ControllerSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/ate-controller", - EgressGatewaySPIFFEID: router, + EgressGatewaySPIFFEIDs: []string{router, "spiffe://cluster.local/ns/plugins/sa/token-exchange"}, IngressRouterSPIFFEID: router, AteletSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/atelet", APIServerSidecarSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/ate-api-server", @@ -106,7 +106,8 @@ func TestParseSystemPrincipals(t *testing.T) { } want := map[string][]string{ "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-controller": {RoleController}, - "user:spiffe%3A//cluster.local/ns/ate-system/sa/atenet-router": {RoleEgressGateway, RoleIngressRouter}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/atenet-router": {RoleIngressRouter, RoleEgressGateway}, + "user:spiffe%3A//cluster.local/ns/plugins/sa/token-exchange": {RoleEgressGateway}, "user:spiffe%3A//cluster.local/ns/ate-system/sa/atelet": {RoleAtelet}, "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-api-server": {RoleAPIServerSidecar}, } @@ -123,13 +124,16 @@ func TestNewRejectsInvalidSystemPrincipals(t *testing.T) { wantErr string }{ {"empty controller SPIFFE ID", func(p *SystemPrincipals) { p.ControllerSPIFFEID = "" }, "invalid controller SPIFFE ID"}, - {"empty egress gateway SPIFFE ID", func(p *SystemPrincipals) { p.EgressGatewaySPIFFEID = "" }, "invalid egress_gateway SPIFFE ID"}, + {"no egress gateway SPIFFE IDs", func(p *SystemPrincipals) { p.EgressGatewaySPIFFEIDs = nil }, "no egress_gateway SPIFFE ID"}, + {"empty egress gateway SPIFFE ID in list", func(p *SystemPrincipals) { + p.EgressGatewaySPIFFEIDs = []string{defaults.EgressGatewaySPIFFEIDs[0], ""} + }, "invalid egress_gateway SPIFFE ID"}, {"empty ingress router SPIFFE ID", func(p *SystemPrincipals) { p.IngressRouterSPIFFEID = "" }, "invalid ingress_router SPIFFE ID"}, {"empty atelet SPIFFE ID", func(p *SystemPrincipals) { p.AteletSPIFFEID = "" }, "invalid atelet SPIFFE ID"}, {"empty api server sidecar SPIFFE ID", func(p *SystemPrincipals) { p.APIServerSidecarSPIFFEID = "" }, "invalid api_server_sidecar SPIFFE ID"}, {"bare ServiceAccount name", func(p *SystemPrincipals) { p.ControllerSPIFFEID = "ate-controller" }, "invalid controller SPIFFE ID"}, {"non-spiffe URI", func(p *SystemPrincipals) { - p.EgressGatewaySPIFFEID = "https://cluster.local/ns/ate-system/sa/atenet-egress" + p.EgressGatewaySPIFFEIDs = []string{"https://cluster.local/ns/ate-system/sa/atenet-egress"} }, "invalid egress_gateway SPIFFE ID"}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/authz/interceptor_test.go b/cmd/ateapi/internal/authz/interceptor_test.go index 110ef6bf1d..b20bd6a7a6 100644 --- a/cmd/ateapi/internal/authz/interceptor_test.go +++ b/cmd/ateapi/internal/authz/interceptor_test.go @@ -626,7 +626,7 @@ func TestUnaryServerInterceptor_SystemPrincipals(t *testing.T) { ) authorizer, _, err := New(ctx, pool, fgaServer, nil, SystemPrincipals{ ControllerSPIFFEID: controllerID, - EgressGatewaySPIFFEID: egressID, + EgressGatewaySPIFFEIDs: []string{egressID}, IngressRouterSPIFFEID: routerID, AteletSPIFFEID: ateletID, APIServerSidecarSPIFFEID: pluginID, diff --git a/cmd/ateapi/internal/store/atepg/common_test.go b/cmd/ateapi/internal/store/atepg/common_test.go index 30a21388ba..c25a3ecbfa 100644 --- a/cmd/ateapi/internal/store/atepg/common_test.go +++ b/cmd/ateapi/internal/store/atepg/common_test.go @@ -57,7 +57,7 @@ func setTestPolicyManager(t *testing.T, p *Persistence) { t.Cleanup(fgaServer.Close) _, policyManager, err := authz.New(t.Context(), p.pool, fgaServer, nil, authz.SystemPrincipals{ ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEIDs: []string{installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace)}, IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), diff --git a/cmd/ateapi/internal/store/storetest/storetest.go b/cmd/ateapi/internal/store/storetest/storetest.go index 7bc219b3d2..2d0f4a5dff 100644 --- a/cmd/ateapi/internal/store/storetest/storetest.go +++ b/cmd/ateapi/internal/store/storetest/storetest.go @@ -101,7 +101,7 @@ func SetupPostgresPersistence(t *testing.T) *atepg.Persistence { func DefaultSystemPrincipals() authz.SystemPrincipals { return authz.SystemPrincipals{ ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEIDs: []string{installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace)}, IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 8e9d74d6f8..9ade6ee253 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -94,7 +94,7 @@ var ( // These SPIFFE IDs hold the matching system component role when they call // over mTLS. controllerSPIFFEID = pflag.String("controller-spiffe-id", installdefaults.ControllerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of ate-controller. Holds the controller authz role, which manages workers.") - egressGatewaySPIFFEID = pflag.String("egress-gateway-spiffe-id", installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-egress. Holds the egress_gateway authz role, which reads and lists actors and reads their egress policies.") + egressGatewaySPIFFEIDs = pflag.StringArray("egress-gateway-spiffe-id", []string{installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv())}, "mTLS SPIFFE IDs of atenet-egress and any other client that holds the egress_gateway authz role, which reads and lists actors and reads their egress policies. Repeat the flag for each ID.") ingressRouterSPIFFEID = pflag.String("ingress-router-spiffe-id", installdefaults.RouterSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-router. Holds the ingress_router authz role, which resumes and lists actors.") apiServerSidecarSPIFFEID = pflag.String("api-server-sidecar-spiffe-id", installdefaults.APIServerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of the ate-api-server pod, which its sidecars share. Holds the api_server_sidecar role, which reads actors so the objectstore control plugin can scope their storage credentials.") @@ -257,7 +257,7 @@ func main() { defer fgaServer.Close() authorizer, policyManager, err := authz.New(shutdownCtx, pool, fgaServer, *authzBootstrapOwners, authz.SystemPrincipals{ ControllerSPIFFEID: *controllerSPIFFEID, - EgressGatewaySPIFFEID: *egressGatewaySPIFFEID, + EgressGatewaySPIFFEIDs: *egressGatewaySPIFFEIDs, IngressRouterSPIFFEID: *ingressRouterSPIFFEID, AteletSPIFFEID: ateletSPIFFEID, APIServerSidecarSPIFFEID: *apiServerSidecarSPIFFEID, diff --git a/docs/authorization.md b/docs/authorization.md index 98ac56ff02..fcf5c5b2a3 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -135,7 +135,7 @@ through their pod certificates. `ate-api-server`'s `--controller-spiffe-id`, | Role | Held by | Allows | |---|---|---| | `controller` | `ate-controller` | Create, get, list, update, delete and drain workers | -| `egress_gateway` | `atenet-egress` | Get actors and their egress policies; list actors across all atespaces; mint actor JWTs and certificates | +| `egress_gateway` | `atenet-egress`, and any other SPIFFE IDs passed with `--egress-gateway-spiffe-id`, which ate-setup sets from `ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS` | Get actors and their egress policies; list actors across all atespaces; mint actor JWTs and certificates | | `ingress_router` | `atenet-router` | Resume actors; list actors across all atespaces | | `atelet` | `atelet` | Get actors' egress policies, for the worker that enforces them | | `api_server_sidecar` | the `ate-api-server` pod identity, which its sidecars share, today the objectstore control plugin | Get actors, to derive the snapshot prefixes a per-actor storage credential may cover | diff --git a/docs/operator-install.md b/docs/operator-install.md index 2cec3c3054..5c8616e043 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -273,6 +273,7 @@ demos, the [benchmarks](../benchmarking/README.md#settings), and the | `actorJWT.algorithm` | `ACTOR_JWT_ALGORITHM` | `--actor-jwt-algorithm` | `ES256` | (global) | | `actorJWT.issuer` | `ACTOR_JWT_ISSUER` | `--actor-jwt-issuer` | — | (global) | | `ateapi.authz.bootstrapOwners` | `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` | `--ateapi-authz-bootstrap-owners` | `system:serviceaccount::ate-client` | (global) | +| `ateapi.authz.egressGatewayExtraSPIFFEIDs` | `ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS` | `--egress-gateway-extra-spiffe-ids` | — | (global) | | `ateapi.expectedJWTIssuer` | `EXPECTED_JWT_ISSUER` | `--ateapi-expected-jwt-issuer` | — | (global) | | `ateapi.logLevel` | `ATE_API_LOG_LEVEL` | `--ateapi-log-level` | — | (global) | | `ateapi.postgres.cloudsql.gsa` | `ATE_API_POSTGRES_CLOUDSQL_GSA` | `--ateapi-postgres-cloudsql-gsa` | — | (global) | From c757792a67b1be4d4bf256ed4fdcbe89d0d265ad Mon Sep 17 00:00:00 2001 From: Mofi Rahman Date: Thu, 29 Oct 2026 03:41:13 +0000 Subject: [PATCH 48/53] Add Hugo Docsite (#1974) Fixes #1863 > It's a good idea to open an issue first for discussion. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR Github Pages URL from the fork: https://moficodes.github.io/substrate/ Signed-off-by: Mofi Rahman --- .github/SECURITY.md | 6 +- .github/workflows/site.yaml | 120 + AGENTS.md | 8 +- CONTRIBUTING.md | 131 +- README.md | 50 +- api/ateapi/v1/AGENTS.md | 2 +- benchmarking/README.md | 1020 +---- benchmarking/automation/README.md | 2 +- cmd/ate-setup/commands.md | 4 +- cmd/atenet/README.md | 2 +- cmd/atenet/internal/router/README.md | 6 +- cmd/kubectl-ate/README.md | 364 +- cmd/objectstore-plugin/README.md | 2 +- demos/claude-code-multiplex/README.md | 12 +- demos/counter/README.md | 12 +- demos/egress/README.md | 12 +- demos/jupyter/README.md | 10 + demos/multi-template/README.md | 10 + demos/parking/README.md | 12 +- demos/sandbox/README.md | 10 + docs/api-style-guide.md | 6 +- docs/api-validation.md | 4 + docs/code-style-guide.md | 4 + docs/{ => concepts}/architecture.md | 24 +- docs/{ => concepts}/request-parking.md | 8 + docs/{ => concepts}/sandbox-asset-cache.md | 9 + docs/dev/best-practices/logging.md | 2 +- docs/dev/best-practices/metrics.md | 8 +- docs/dev/best-practices/otel-collector.md | 6 +- docs/dev/microvm-local.md | 2 +- docs/dev/suspend-resume-phase-breakdown.md | 2 +- docs/examples/gmp-alerting-rules.yaml | 8 +- docs/get-started/overview.md | 55 + docs/get-started/quickstart.md | 87 + docs/{ => guides}/authentication.md | 8 + docs/{ => guides}/authorization.md | 12 +- docs/guides/benchmarking.md | 1028 +++++ docs/{ => guides}/csi-deployment.md | 10 +- docs/{ => guides}/csi-volumes.md | 14 +- .../egress-credential-injection.md | 10 +- docs/{ => guides}/egress-traffic.md | 12 + docs/{ => guides}/egress-trust-bundle.md | 12 +- docs/{ => guides}/network-egress.md | 10 +- docs/{ => guides}/node-storage.md | 9 + docs/{ => guides}/observability.md | 99 +- docs/{ => guides}/operator-install.md | 24 +- docs/{ => guides}/postgres.md | 16 +- docs/{ => guides}/slos-and-alerts.md | 16 +- docs/{ => guides}/upgrade.md | 12 +- docs/project/community.md | 39 + docs/project/contributing.md | 141 + docs/{ => project}/integration-repos.md | 11 + docs/{ => reference}/api-guide.md | 31 +- docs/{ => reference}/glossary.md | 23 +- docs/reference/kubectl-ate.md | 362 ++ docs/{ => reference}/resource-model.md | 10 +- docs/{ => reference}/version-skew-policy.md | 14 +- docs/{ => security}/extensibility.md | 14 +- docs/{ => security}/security.md | 14 +- docs/{ => security}/threat-model.md | 10 +- hack/export-threats.py | 2 +- hack/site.sh | 80 + hack/test-site-templates.sh | 219 + hack/util/verify-boilerplate.py | 2 +- hack/verify/threats.sh | 2 +- internal/e2e/README.md | 2 +- site/.gitignore | 6 + site/.npmrc | 16 + site/README.md | 99 + site/assets/icons/logo.png | Bin 0 -> 380857 bytes site/assets/js/click-to-copy.js | 587 +++ site/assets/js/glossary.js | 172 + site/assets/js/offline-search.js | 290 ++ site/assets/scss/_styles_project.scss | 3844 +++++++++++++++++ site/assets/scss/_variables_project.scss | 54 + site/go.mod | 5 + site/go.sum | 4 + site/hugo.yaml | 172 + site/layouts/404.html | 123 + site/layouts/_markup/render-blockquote.html | 33 + site/layouts/_partials/favicons.html | 31 + site/layouts/_partials/feedback.html | 61 + site/layouts/_partials/footer.html | 84 + site/layouts/_partials/glossary-terms.html | 70 + site/layouts/_partials/hooks/body-end.html | 35 + site/layouts/_partials/hooks/head-end.html | 20 + site/layouts/_partials/navbar.html | 56 + site/layouts/_partials/page-meta-links.html | 17 + site/layouts/_partials/page-params.html | 34 + site/layouts/_partials/page-pills.html | 35 + site/layouts/_partials/pager.html | 85 + site/layouts/_partials/raw-markdown.html | 58 + site/layouts/_partials/repo-doc.html | 50 + .../layouts/_partials/rewrite-repo-links.html | 65 + site/layouts/_partials/section-index.html | 40 + site/layouts/_partials/sidebar.html | 45 + site/layouts/_partials/status-text.html | 32 + site/layouts/_partials/toc.html | 123 + site/layouts/_shortcodes/blocks/cover.html | 37 + site/layouts/_shortcodes/home.html | 722 ++++ site/layouts/_shortcodes/include-file.html | 46 + site/layouts/docs/list.html | 36 + site/layouts/docs/single.html | 62 + site/layouts/index.llms.txt | 39 + site/layouts/index.llmsfull.txt | 38 + site/package-lock.json | 696 +++ site/package.json | 10 + site/static/CNAME | 1 + site/testdata/content-bogus/_index.md | 3 + site/testdata/content-bogus/docs/_index.md | 3 + .../testdata/content-bogus/docs/bad/_index.md | 3 + site/testdata/content-bogus/docs/bad/bogus.md | 6 + site/testdata/content-glossary/_index.md | 3 + site/testdata/content-glossary/docs/_index.md | 3 + .../content-glossary/docs/reference/_index.md | 4 + .../docs/reference/glossary.md | 6 + site/testdata/content/_index.md | 3 + site/testdata/content/docs/_index.md | 3 + site/testdata/content/docs/fixtures/_index.md | 6 + .../content/docs/fixtures/deprecated.md | 8 + .../content/docs/fixtures/experimental.md | 8 + .../content/docs/fixtures/new-deprecated.md | 8 + site/testdata/content/docs/fixtures/new.md | 8 + .../content/docs/fixtures/override.md | 9 + site/testdata/content/docs/fixtures/plain.md | 8 + .../testdata/content/docs/fixtures/planned.md | 8 + .../testdata/content/docs/reference/_index.md | 4 + .../content/docs/reference/glossary.md | 6 + site/testdata/data-empty/placeholder.json | 1 + site/testdata/data/release.json | 5 + .../glossary-errors/codeterm/docs/glossary.md | 3 + .../duplicate/docs/glossary.md | 5 + .../glossary-errors/empty/docs/glossary.md | 3 + site/testdata/hugo.bogus.yaml | 31 + site/testdata/hugo.nodata.yaml | 33 + site/testdata/hugo.test.yaml | 32 + site/testdata/js/glossary.test.js | 83 + site/testdata/repo/docs/bogus.md | 7 + site/testdata/repo/docs/deprecated.md | 8 + site/testdata/repo/docs/experimental.md | 7 + site/testdata/repo/docs/glossary.md | 21 + site/testdata/repo/docs/new-deprecated.md | 7 + site/testdata/repo/docs/new.md | 3 + site/testdata/repo/docs/override.md | 7 + site/testdata/repo/docs/plain.md | 8 + site/testdata/repo/docs/planned.md | 7 + tools/setup-gcp/cloud-sql.md | 2 +- tools/site-content/main.go | 359 ++ tools/site-content/main_test.go | 109 + tools/site-release-data/main.go | 154 + tools/site-release-data/main_test.go | 182 + 151 files changed, 11819 insertions(+), 1689 deletions(-) create mode 100644 .github/workflows/site.yaml rename docs/{ => concepts}/architecture.md (96%) rename docs/{ => concepts}/request-parking.md (98%) rename docs/{ => concepts}/sandbox-asset-cache.md (98%) create mode 100644 docs/get-started/overview.md create mode 100644 docs/get-started/quickstart.md rename docs/{ => guides}/authentication.md (95%) rename docs/{ => guides}/authorization.md (98%) create mode 100644 docs/guides/benchmarking.md rename docs/{ => guides}/csi-deployment.md (97%) rename docs/{ => guides}/csi-volumes.md (91%) rename docs/{ => guides}/egress-credential-injection.md (98%) rename docs/{ => guides}/egress-traffic.md (92%) rename docs/{ => guides}/egress-trust-bundle.md (95%) rename docs/{ => guides}/network-egress.md (89%) rename docs/{ => guides}/node-storage.md (83%) rename docs/{ => guides}/observability.md (89%) rename docs/{ => guides}/operator-install.md (97%) rename docs/{ => guides}/postgres.md (88%) rename docs/{ => guides}/slos-and-alerts.md (97%) rename docs/{ => guides}/upgrade.md (98%) create mode 100644 docs/project/community.md create mode 100644 docs/project/contributing.md rename docs/{ => project}/integration-repos.md (96%) rename docs/{ => reference}/api-guide.md (97%) rename docs/{ => reference}/glossary.md (90%) create mode 100644 docs/reference/kubectl-ate.md rename docs/{ => reference}/resource-model.md (93%) rename docs/{ => reference}/version-skew-policy.md (95%) rename docs/{ => security}/extensibility.md (66%) rename docs/{ => security}/security.md (91%) rename docs/{ => security}/threat-model.md (99%) create mode 100755 hack/site.sh create mode 100755 hack/test-site-templates.sh create mode 100644 site/.gitignore create mode 100644 site/.npmrc create mode 100644 site/README.md create mode 100644 site/assets/icons/logo.png create mode 100644 site/assets/js/click-to-copy.js create mode 100644 site/assets/js/glossary.js create mode 100644 site/assets/js/offline-search.js create mode 100644 site/assets/scss/_styles_project.scss create mode 100644 site/assets/scss/_variables_project.scss create mode 100644 site/go.mod create mode 100644 site/go.sum create mode 100644 site/hugo.yaml create mode 100644 site/layouts/404.html create mode 100644 site/layouts/_markup/render-blockquote.html create mode 100644 site/layouts/_partials/favicons.html create mode 100644 site/layouts/_partials/feedback.html create mode 100644 site/layouts/_partials/footer.html create mode 100644 site/layouts/_partials/glossary-terms.html create mode 100644 site/layouts/_partials/hooks/body-end.html create mode 100644 site/layouts/_partials/hooks/head-end.html create mode 100644 site/layouts/_partials/navbar.html create mode 100644 site/layouts/_partials/page-meta-links.html create mode 100644 site/layouts/_partials/page-params.html create mode 100644 site/layouts/_partials/page-pills.html create mode 100644 site/layouts/_partials/pager.html create mode 100644 site/layouts/_partials/raw-markdown.html create mode 100644 site/layouts/_partials/repo-doc.html create mode 100644 site/layouts/_partials/rewrite-repo-links.html create mode 100644 site/layouts/_partials/section-index.html create mode 100644 site/layouts/_partials/sidebar.html create mode 100644 site/layouts/_partials/status-text.html create mode 100644 site/layouts/_partials/toc.html create mode 100644 site/layouts/_shortcodes/blocks/cover.html create mode 100644 site/layouts/_shortcodes/home.html create mode 100644 site/layouts/_shortcodes/include-file.html create mode 100644 site/layouts/docs/list.html create mode 100644 site/layouts/docs/single.html create mode 100644 site/layouts/index.llms.txt create mode 100644 site/layouts/index.llmsfull.txt create mode 100644 site/package-lock.json create mode 100644 site/package.json create mode 100644 site/static/CNAME create mode 100644 site/testdata/content-bogus/_index.md create mode 100644 site/testdata/content-bogus/docs/_index.md create mode 100644 site/testdata/content-bogus/docs/bad/_index.md create mode 100644 site/testdata/content-bogus/docs/bad/bogus.md create mode 100644 site/testdata/content-glossary/_index.md create mode 100644 site/testdata/content-glossary/docs/_index.md create mode 100644 site/testdata/content-glossary/docs/reference/_index.md create mode 100644 site/testdata/content-glossary/docs/reference/glossary.md create mode 100644 site/testdata/content/_index.md create mode 100644 site/testdata/content/docs/_index.md create mode 100644 site/testdata/content/docs/fixtures/_index.md create mode 100644 site/testdata/content/docs/fixtures/deprecated.md create mode 100644 site/testdata/content/docs/fixtures/experimental.md create mode 100644 site/testdata/content/docs/fixtures/new-deprecated.md create mode 100644 site/testdata/content/docs/fixtures/new.md create mode 100644 site/testdata/content/docs/fixtures/override.md create mode 100644 site/testdata/content/docs/fixtures/plain.md create mode 100644 site/testdata/content/docs/fixtures/planned.md create mode 100644 site/testdata/content/docs/reference/_index.md create mode 100644 site/testdata/content/docs/reference/glossary.md create mode 100644 site/testdata/data-empty/placeholder.json create mode 100644 site/testdata/data/release.json create mode 100644 site/testdata/glossary-errors/codeterm/docs/glossary.md create mode 100644 site/testdata/glossary-errors/duplicate/docs/glossary.md create mode 100644 site/testdata/glossary-errors/empty/docs/glossary.md create mode 100644 site/testdata/hugo.bogus.yaml create mode 100644 site/testdata/hugo.nodata.yaml create mode 100644 site/testdata/hugo.test.yaml create mode 100644 site/testdata/js/glossary.test.js create mode 100644 site/testdata/repo/docs/bogus.md create mode 100644 site/testdata/repo/docs/deprecated.md create mode 100644 site/testdata/repo/docs/experimental.md create mode 100644 site/testdata/repo/docs/glossary.md create mode 100644 site/testdata/repo/docs/new-deprecated.md create mode 100644 site/testdata/repo/docs/new.md create mode 100644 site/testdata/repo/docs/override.md create mode 100644 site/testdata/repo/docs/plain.md create mode 100644 site/testdata/repo/docs/planned.md create mode 100644 tools/site-content/main.go create mode 100644 tools/site-content/main_test.go create mode 100644 tools/site-release-data/main.go create mode 100644 tools/site-release-data/main_test.go diff --git a/.github/SECURITY.md b/.github/SECURITY.md index d5acdcc747..7315142a06 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -37,8 +37,8 @@ include: ## What Counts as a Vulnerability -See the [Security Guide](../docs/security.md) and -[Threat Model](../docs/threat-model.md) for the security architecture, trust +See the [Security Guide](../docs/security/security.md) and +[Threat Model](../docs/security/threat-model.md) for the security architecture, trust boundaries, and assumptions this section is based on. In scope: the Agent Substrate control plane (`ateapi`, `atecontroller`, @@ -81,7 +81,7 @@ from most to least severe: - Sample applications under `demos/` and load-testing harnesses under `benchmarking/`. - Documented scope boundaries and future capabilities listed in - [docs/security.md](../docs/security.md) at the time of your report. If you are + [docs/security/security.md](../docs/security/security.md) at the time of your report. If you are unsure whether a finding is covered by a documented limitation, report it privately anyway and let a maintainer make the call. diff --git a/.github/workflows/site.yaml b/.github/workflows/site.yaml new file mode 100644 index 0000000000..efca938169 --- /dev/null +++ b/.github/workflows/site.yaml @@ -0,0 +1,120 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds the website in site/ on every pull request that could change it, and +# publishes it to GitHub Pages from main. +name: site +on: + pull_request: + paths: + - 'site/**' + - 'docs/**' + - 'demos/*/README.md' + - 'tools/site-content/**' + - 'tools/site-release-data/**' + - 'hack/site.sh' + - 'hack/test-site-templates.sh' + - '.github/workflows/site.yaml' + push: + branches: [main] + paths: + - 'site/**' + - 'docs/**' + - 'demos/*/README.md' + - 'tools/site-content/**' + - 'tools/site-release-data/**' + - 'hack/site.sh' + - 'hack/test-site-templates.sh' + - '.github/workflows/site.yaml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +env: + # Keep in sync with module.hugoVersion.min in site/hugo.yaml. + HUGO_VERSION: 0.163.3 +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + # site-release-data needs the latest release tag and its history. + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + # The root module, for tools/site-release-data. Hugo fetches Docsy + # with the same Go. + go-version-file: 'go.mod' + cache: false + - name: Setup Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + cache: npm + cache-dependency-path: site/package-lock.json + - name: Install Hugo + run: | + base="https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}" + tarball="hugo_extended_${HUGO_VERSION}_linux-amd64.tar.gz" + cd "${RUNNER_TEMP}" + curl -sSfLO "${base}/${tarball}" + curl -sSfLO "${base}/hugo_${HUGO_VERSION}_checksums.txt" + grep " ${tarball}\$" "hugo_${HUGO_VERSION}_checksums.txt" | sha256sum -c - + mkdir -p hugo && tar -xzf "${tarball}" -C hugo hugo + echo "${RUNNER_TEMP}/hugo" >> "${GITHUB_PATH}" + - name: Install PostCSS + working-directory: site + run: npm ci + - name: Generate site content + run: hack/site.sh content + - name: Generate release data + run: hack/site.sh release-data + - name: Test templates + if: github.event_name == 'pull_request' + run: hack/test-site-templates.sh + - name: Configure Pages + id: pages + if: github.event_name != 'pull_request' + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + - name: Build + working-directory: site + env: + BASE_URL: ${{ steps.pages.outputs.base_url }} + run: hugo build --environment production --minify ${BASE_URL:+--baseURL "${BASE_URL}/"} + - name: Upload artifact + if: github.event_name != 'pull_request' + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + path: site/public + deploy: + if: github.event_name != 'pull_request' + needs: build + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/AGENTS.md b/AGENTS.md index 30b87bdd71..3919a29188 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,7 +132,7 @@ Do not add `reserved` statements to protos, deprecated aliases, fallbacks for ol ## Version Skew -From v1.0.0, changes must keep upgrades and rollbacks working under [docs/version-skew-policy.md](docs/version-skew-policy.md). Read it before you change an RPC, a CRD, the worker pod template, a database migration, or file(s) that `atelet` and `ateworker` share on a node. +From v1.0.0, changes must keep upgrades and rollbacks working under [docs/reference/version-skew-policy.md](docs/reference/version-skew-policy.md). Read it before you change an RPC, a CRD, the worker pod template, a database migration, or file(s) that `atelet` and `ateworker` share on a node. ## Commit Messages @@ -149,7 +149,7 @@ Some attributes are defined in the registry but barred from metric labels: actor `docs/metrics/substrate.yaml` holds the rules Weaver cannot express: the cardinality rules, the known exceptions, and the subsystems that emit no metrics. Read `blind_spots` before you attribute a fault to a component. -See the [metric registry](docs/observability.md#the-metric-registry) section of the observability guide. +See the [metric registry](docs/guides/observability.md#the-metric-registry) section of the observability guide. ## Testing Instructions @@ -161,9 +161,9 @@ See the [metric registry](docs/observability.md#the-metric-registry) section of ## Security Considerations -Agent Substrate is built for zero-trust, multi-tenant execution of untrusted actors. Preserve and extend these invariants when modifying the codebase (see `docs/security.md` and `docs/threat-model.md` for the full architecture): +Agent Substrate is built for zero-trust, multi-tenant execution of untrusted actors. Preserve and extend these invariants when modifying the codebase (see `docs/security/security.md` and `docs/security/threat-model.md` for the full architecture): - **Workload & Storage Isolation**: Actors run strictly inside `gVisor` (`cmd/ateworker-gvisor`) or `Cloud Hypervisor` microVM (`cmd/ateworker-microvm`) sandboxes; plain Linux containers are not supported. `atelet` scrubs actor directories across lifecycle transitions (`resetActorDirs`), and snapshots are stamped with `ActorSandboxRuntime` metadata and transferred via `cmd/objectstore-plugin`. - **Component & Actor Identity (mTLS, JWTs, and JWKs)**: Inter-component communication uses mTLS backed by `cmd/podcertcontroller` (`podid`, `servicedns`, and `postgres` signers). Actor JWT signing (`internal/localjwtauthority`), verification (`internal/oidcjwt`), and key discovery (served via `cmd/ate-idp-server`) are built on the standard library's crypto packages. Extend those packages rather than adding `go-jose` or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses. -- **Control-Plane Authentication & Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), always enforced. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/authorization.md` for roles and known gaps. Callers are authenticated via `SPIFFE` mTLS or configured OIDC `JWTProvider`s (`cmd/ateapi/internal/apiauthn`). +- **Control-Plane Authentication & Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), always enforced. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/guides/authorization.md` for roles and known gaps. Callers are authenticated via `SPIFFE` mTLS or configured OIDC `JWTProvider`s (`cmd/ateapi/internal/apiauthn`). - **Network Isolation, Ingress, & Egress**: `manifests/ate-install/ate-system-networkpolicies.yaml` enforces default-deny ingress across `ate-system`, and `cmd/atecontroller` reconciles per-`WorkerPool` `NetworkPolicy` objects. Actors run in private `/30` point-to-point veth network namespaces (`internal/ateworker/actornet`) with link-local blocking, policy-enforced DNS filtering (`internal/atunnel/dns`), and zero ambient credentials. Outbound traffic traverses `atunnel` to `atenet-egress`, which enforces per-actor `EgressPolicy` and injects credentials out-of-band via `cmd/credential-provider/kubernetes-secrets` (`k8s-credential-provider`) or actor JWT minting. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 022d65cb3e..996d14f619 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,132 +1,3 @@ # How to Contribute -We would love to accept your patches and contributions to this project. Before -you spend a lot of time on a contribution, please review the following -guidelines. - -## Before you begin - -### Sign off your commits - -Every commit must be signed off under the -[Developer Certificate of Origin](https://developercertificate.org/) (DCO). -The `Signed-off-by` line certifies that you wrote the change or otherwise have -the right to submit it under the project's license. - -Sign off with your real name, not an anonymous ID or a false name. It doesn't -need to be your legal name, just the name the community knows you by. See the -[CNCF DCO guidelines](https://github.com/cncf/foundation/blob/main/policies-guidance/dco-guidelines.md). - -Git adds the line with `-s`, using `user.name` and `user.email` from your git -config. The name and email must match the commit author. - -```sh -git commit -s -m "fix typo in README" -``` - -The DCO check on your PR fails if any commit is missing a sign-off. To sign off -the commits already on your branch, then force-push: - -```sh -git rebase --signoff upstream/main -``` - -### Review our Community Guidelines - -This project follows [Google's Open Source Community -Guidelines](https://opensource.google/conduct/). - -### Set up a local development environment - -The [Quickstart (Development)](README.md#quickstart-development) in the README -covers bringing up a local cluster with the default (gVisor) runtime. To run -the microVM runtime locally — which needs `/dev/kvm`, or Lima nested -virtualization on Apple Silicon — see -[docs/dev/microvm-local.md](docs/dev/microvm-local.md). To measure where a -suspend or resume spends its time on such a cluster, follow -[docs/dev/suspend-resume-phase-breakdown.md](docs/dev/suspend-resume-phase-breakdown.md). - -## Contribution process - -This is a very new project, so we are still working out exactly how it is going -to be developed. For now, we are focused on iterating quickly to find the right -design and architecture. This has implications for contributors: - -1) Things are moving quickly, so PRs may need to be rebased or updated -frequently. Small PRs that are focused on a single issue or feature are easier -to review and update than large PRs that touch many different parts of the -codebase. - -2) While we welcome new contributors, we are really focused on the minimal -capabilities needed to make this project useful. Small fixes and cleanups can -go straight to a PR. Before you start anything larger, please discuss it with -us first (if there is an issue open, comment there and if not, open one), and -search open PRs so we are not duplicating work that is already in flight. We -want to make sure that your work is aligned with our near-term goals for the -project. - -3) PRs which are not aligned with our near-term goals may be closed without -extensive review. We are not trying to be discouraging, but we need to make -sure that we are focused on the most important work. - -If you are building something that runs *on* Substrate rather than changing -Substrate itself, see [Integration -Repositories](docs/integration-repos.md) for where that code should live. - -### Sizing PRs for review - -We optimize PRs for easy review — large PRs get broken -down, small PRs get merged. - -* **Large PRs**: split huge changes into a series of smaller PRs, each a - logically distinct feature. When the intermediate steps are not useful - on their own, keep the change as one PR split into commits at logical - break points for review. PRs are squash-merged, so only the branch - `HEAD` needs to build and pass tests. -* **Small and bulk PRs**: if you find a typo, review the whole file and - fix everything in one pass rather than sending the single edit. Group - related typo, doc, and single-line cleanup fixes into one PR rather - than opening several small ones for the same area. Maintainers may ask - you to consolidate fragmented PRs into one, or close them in favor of a - combined submission. - -As a rough scale: S is under 30 changed lines, M under 100, L under 500, XL under 1000. Most PRs should be L or smaller; XL and above are candidates for breaking down. - -### Code Reviews - -All submissions, including submissions by project members, require review. We -use [GitHub pull requests](https://docs.github.com/articles/about-pull-requests) -for this purpose. - -Changes that add or alter behavior should come with tests, and existing tests -should be updated when behavior they cover changes. We will not merge code that -causes tests to fail. Every test runs in CI on every PR, so please don't add -tests whose only purpose is to raise coverage numbers. Tests should be -inherently valuable for future changes. - -Follow the [PostgreSQL schema evolution rules](docs/dev/postgresql-schema-evolution.md) for each application schema change. - -Each pull request title becomes its line in the [release notes](docs/dev/release-notes.md). - -### Root-gated tests - -Tests that need root (overlay mounts, mknod, `trusted.*` xattrs, ...) call -`roottest.Require(t, ...)` from [internal/roottest](internal/roottest) as -their first statement. They skip in a plain `go test ./...`; CI reruns every -package whose tests import that package under `sudo`. To run them locally: - -```sh -hack/run-root-tests.sh -``` - -New privileged tests only need the `roottest.Require` call — no CI changes. - -### Copyright Headers - -Every file containing source code must include copyright and license -information. This includes any JS/CSS files that you might be serving out to -browsers. (This is to help well-intentioned people avoid accidental copying -that doesn't comply with the license.) - -Our standard headers for various filetypes can be found in -[./hack/boilerplate](./hack/boilerplate/). +Moved to [docs/project/contributing.md](docs/project/contributing.md). diff --git a/README.md b/README.md index 6e89e288e6..d6dd7a40ea 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[Quickstart](#quickstart-development) · [Security](docs/security.md) · [Architecture](docs/architecture.md) · [Contributing](CONTRIBUTING.md) +[Quickstart](#quickstart-development) · [Security](docs/security/security.md) · [Architecture](docs/concepts/architecture.md) · [Contributing](CONTRIBUTING.md) @@ -25,12 +25,12 @@ Substrate is a low-opinion runtime, not an agent SDK. The workloads it runs do n ## Security Model -Substrate treats every actor as untrusted and mutually hostile, so tenants can share physical nodes without sharing a security boundary. Every control operates within the cold-start and resume budget. See the [Security Guide](docs/security.md) for the full list of security principles and threat model. +Substrate treats every actor as untrusted and mutually hostile, so tenants can share physical nodes without sharing a security boundary. Every control operates within the cold-start and resume budget. See the [Security Guide](docs/security/security.md) for the full list of security principles and threat model. * **Always sandboxed, VM-grade boundaries:** Every actor runs inside a dedicated [gVisor](https://gvisor.dev/) (`runsc`) or [Cloud Hypervisor](https://www.cloudhypervisor.org/) microVM sandbox with segregated compute, network, and storage. Plain Linux containers are not supported. * **Per-agent cryptographic identity:** Every active actor receives a short-lived [SPIFFE](https://spiffe.io/) X.509 certificate by default, and the egress gateway can mint per-actor JWTs on demand when an egress policy calls for one. Both carry the actor identity and atespace (the Substrate namespace grouping), and are designed to federate with external identity systems. -* **Zero credentials in the sandbox:** Actors never hold actor identity JWTs/certs, API keys, or cloud tokens. The built-in egress gateway ([`atenet-egress`](docs/egress-credential-injection.md)) injects credentials into approved outbound requests outside the sandbox, so a prompt-injected or compromised agent has nothing to exfiltrate. -* **Default-deny networking:** Actors start with a private network stack, no inbound path except through the ingress router (`atenet-router`), no lateral actor-to-actor or control-plane access, and outbound traffic [restricted by the egress gateway](docs/egress-traffic.md). +* **Zero credentials in the sandbox:** Actors never hold actor identity JWTs/certs, API keys, or cloud tokens. The built-in egress gateway ([`atenet-egress`](docs/guides/egress-credential-injection.md)) injects credentials into approved outbound requests outside the sandbox, so a prompt-injected or compromised agent has nothing to exfiltrate. +* **Default-deny networking:** Actors start with a private network stack, no inbound path except through the ingress router (`atenet-router`), no lateral actor-to-actor or control-plane access, and outbound traffic [restricted by the egress gateway](docs/guides/egress-traffic.md). * **Pluggable policy at the edge:** The egress gateway acts as an external Policy Enforcement Point (PEP) that is fully replaceable, allowing you to attach your own authorization, DLP, prompt-injection scanning, or cloud IAM policy engine without placing policy code inside the untrusted sandbox. ## Performance & Multiplexing @@ -145,7 +145,7 @@ kubectl ate proxy actor my-counter-1 -a ate-demo-counter > `{"name":"k8s.io"}` deploys the bundled credential provider with an empty, > default-deny namespace policy. Before an actor can have a credential > injected, grant its atespace the namespaces whose Secrets it may read; see -> [Enable it](docs/egress-credential-injection.md#enable-it). +> [Enable it](docs/guides/egress-credential-injection.md#enable-it). 3. In a **separate terminal**, send an HTTP request to increment the counter: ```shell @@ -190,18 +190,18 @@ curl -X POST -i http://localhost:8000/ `--ingress-allowed-spiffe-ids` lists the clients the router lets reach actors. The ID above is the one `kubectl ate proxy actor` presents. Without the flag no client reaches actors. See - [Ingress to actors](docs/operator-install.md#ingress-to-actors). + [Ingress to actors](docs/guides/operator-install.md#ingress-to-actors). `--credential-provider` is required. `{"name":"k8s.io"}` also deploys the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off - (see [docs/egress-credential-injection.md](docs/egress-credential-injection.md)). + (see [docs/guides/egress-credential-injection.md](docs/guides/egress-credential-injection.md)). `ATE_CREDENTIAL_PROVIDER` in `.ate-dev-env.sh` is the same choice. **Credential injection needs a namespace policy.** The bundled provider starts with an empty, default-deny one, so it resolves no Secrets until you grant each atespace the namespaces it may read; see - [Enable it](docs/egress-credential-injection.md#enable-it). + [Enable it](docs/guides/egress-credential-injection.md#enable-it). 5. You can then deploy the sample applications. See [demos/counter/README.md](demos/counter/README.md) or [demos/sandbox/README.md](demos/sandbox/README.md) for detailed walkthroughs. ```bash @@ -261,22 +261,22 @@ We provide several sample applications demonstrating Agent Substrate's capabilit 5. **[Request Parking](demos/parking/README.md)**: An oversubscribed pool where the router holds inbound requests until a worker frees up, instead of returning `503`. ### Documentation & Guides -* [Architecture](docs/architecture.md): How the control plane, node supervisor, and networking stack fit together. -* [API Configuration Guide](docs/api-guide.md): Detailed reference for configuring WorkerPools, ActorTemplates, Secrets, and Volumes. -* [Full CLI Documentation](cmd/kubectl-ate/README.md): Installation and usage for `kubectl-ate`. -* [Glossary](docs/glossary.md): Core terms (Actor, Atespace, ActorTemplate, WorkerPool, Worker, ate-api-server, atenet, atelet, ateworker) and how they relate. -* [Integration Repositories](docs/integration-repos.md): Where integrations live, how their repositories are named, and how fixes flow back to core. -* [Observability Guide](docs/observability.md): Guide to actor logging, metrics, and distributed tracing. -* [Extending Substrate](docs/extensibility.md): How to augment or replace the built-in ingress and egress gateways. -* [Authentication Guide](docs/authentication.md): Configure trusted JWT providers and human credentials. -* [Egress Traffic](docs/egress-traffic.md): Which protocols an Actor may reach the outside world with and which are blocked. -* [Enabling man-in-the-middle (MITM) interception for Actor Egress policy](docs/egress-trust-bundle.md): Egress policies such as header injection depend on MITM interception of Actor traffic. This guide explains how an Actor should be configured to enable interception. -* [Request Parking](docs/request-parking.md): How the router parks requests through transient worker-pool saturation. -* [Sandbox Asset Cache](docs/sandbox-asset-cache.md): How a node caches the sandbox binaries actors run on, decides which runtimes it needs, and evicts to a budget. -* [Upgrade Runbook](docs/upgrade.md): Upgrade a running install to a newer supported release, and roll back if needed. -* [Node Storage](docs/node-storage.md): How to give snapshot I/O its own disk. -* [Security Guide](docs/security.md): Security overview, security principles, and threat modeling. -* [Benchmarking Guide](benchmarking/README.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness. +* [Architecture](docs/concepts/architecture.md): How the control plane, node supervisor, and networking stack fit together. +* [API Configuration Guide](docs/reference/api-guide.md): Detailed reference for configuring WorkerPools, ActorTemplates, Secrets, and Volumes. +* [Full CLI Documentation](docs/reference/kubectl-ate.md): Installation and usage for `kubectl-ate`. +* [Glossary](docs/reference/glossary.md): Core terms (Actor, Atespace, ActorTemplate, WorkerPool, Worker, ate-api-server, atenet, atelet, ateworker) and how they relate. +* [Integration Repositories](docs/project/integration-repos.md): Where integrations live, how their repositories are named, and how fixes flow back to core. +* [Observability Guide](docs/guides/observability.md): Guide to actor logging, metrics, and distributed tracing. +* [Extending Substrate](docs/security/extensibility.md): How to augment or replace the built-in ingress and egress gateways. +* [Authentication Guide](docs/guides/authentication.md): Configure trusted JWT providers and human credentials. +* [Egress Traffic](docs/guides/egress-traffic.md): Which protocols an Actor may reach the outside world with and which are blocked. +* [Enabling man-in-the-middle (MITM) interception for Actor Egress policy](docs/guides/egress-trust-bundle.md): Egress policies such as header injection depend on MITM interception of Actor traffic. This guide explains how an Actor should be configured to enable interception. +* [Request Parking](docs/concepts/request-parking.md): How the router parks requests through transient worker-pool saturation. +* [Sandbox Asset Cache](docs/concepts/sandbox-asset-cache.md): How a node caches the sandbox binaries actors run on, decides which runtimes it needs, and evicts to a budget. +* [Upgrade Runbook](docs/guides/upgrade.md): Upgrade a running install to a newer supported release, and roll back if needed. +* [Node Storage](docs/guides/node-storage.md): How to give snapshot I/O its own disk. +* [Security Guide](docs/security/security.md): Security overview, security principles, and threat modeling. +* [Benchmarking Guide](docs/guides/benchmarking.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness. ## Tour @@ -291,7 +291,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * `cmd/ateworker-microvm`: The microVM peer of `ateworker-gvisor`, running actors as cloud-hypervisor VMs. * `cmd/podcertcontroller`: A "polyfill" that provides Pod Certificate signers that will eventually ship in upstream Kubernetes (with different names). -* `cmd/kubectl-ate`: A CLI tool for managing Agent Substrate resources. See its [README](cmd/kubectl-ate/README.md). +* `cmd/kubectl-ate`: A CLI tool for managing Agent Substrate resources. See its [README](docs/reference/kubectl-ate.md). * `cmd/benchmarking`: Synthetic workloads used by the load tests, including `glutton`, which consumes RAM, disk, and file descriptors on demand. * `tools/setup-gcp`: A provisioning utility to set up the necessary GCP infrastructure resources (GKE, GCS, IAM). * `demos/`: Sample applications demonstrating Agent Substrate capabilities. diff --git a/api/ateapi/v1/AGENTS.md b/api/ateapi/v1/AGENTS.md index 511e64317d..d90bfa3edd 100644 --- a/api/ateapi/v1/AGENTS.md +++ b/api/ateapi/v1/AGENTS.md @@ -3,6 +3,6 @@ `ateapi.proto` defines Substrate's public API. - See `docs/api-style-guide.md` for the API style guide. -- See `docs/resource-model.md` for the resources and subresources the API exposes. Update it when you add, remove, or re-parent a resource. +- See `docs/reference/resource-model.md` for the resources and subresources the API exposes. Update it when you add, remove, or re-parent a resource. - See `docs/api-validation.md` for how to use declarative validation (DV) tags. - `*.pb.go` files are generated. Never edit them by hand. After changing `ateapi.proto`, run `hack/update/codegen.sh`, which also regenerates the DV code in `cmd/ateapi/internal/apivalidation`. diff --git a/benchmarking/README.md b/benchmarking/README.md index 803989d54b..50a9d9d397 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -1,1019 +1 @@ -# Substrate Benchmarking - -This is the nascent suite for benchmarking Substrate's performance at scale. - -The suite also measures the telemetry volume and the capacity of the OTel -collector: how much trace data and metric data substrate and its actors send, -and if the collector can accept it. To make a measurement, read -[telemetry/README.md](telemetry/README.md). For the prerequisites and the -scenario ladder, read [observability.md](observability.md). - -## Deploy benchmarks - -> [!IMPORTANT] -> Source the environment configuration file (e.g., `source .ate-dev-env.sh`) -> first so `PROJECT_ID` etc. are set. -> -> **For Local Kind Clusters**: Set `KO_DOCKER_REPO` to the local registry so images are pulled without remote cloud credentials: -> ```bash -> export KO_DOCKER_REPO="localhost:5001" -> ``` - -> [!IMPORTANT] -> The ate-api-server checks every call against -> [authorization](../docs/authorization.md), and Locust calls it as the -> `benchmarking/default` ServiceAccount. The Envoy ingress router also requires -> an allowlist of client SPIFFE IDs at install time. Install Substrate with -> `benchmarking/default` in both settings alongside the default `ate-client` -> owner: -> ```bash -> ATE_API_AUTHZ_BOOTSTRAP_OWNERS=system:serviceaccount:ate-system:ate-client,spiffe://cluster.local/ns/benchmarking/sa/default \ -> ATE_INGRESS_ALLOWED_SPIFFE_IDS=spiffe://cluster.local/ns/benchmarking/sa/default \ -> go run ./cmd/ate-setup deploy ate-system -> ``` -> The nightly automation sets both itself; see -> [automation/README.md](automation/README.md). - -Note that deploying the benchmarks does not run them. You must visit Locust's -web UI to start a test. - -A single wrapper deploys the scale workloads, builds and pushes the Locust -image, then deploys the Locust workers: - -```bash -./benchmarking/deploy_locust.sh --deploy -``` - -Useful flags: - -* `--worker-count N` — number of `WorkerPool` replicas (default 1). -* `--worker-memory SIZE` — memory request and limit for each `WorkerPool` pod - (default unset, the pod is unsized). Set it above half a node's allocatable - memory to get one worker per node. -* `--skip-build` — reuse the existing `:latest` locust image (skip the - `docker build && docker push` step). - -To tear everything down (locust then workloads, in reverse order): - -```bash -./benchmarking/deploy_locust.sh --delete -``` - -The same operations are also reachable from the top-level installer for -convenience: - -```bash -go run ./cmd/ate-setup deploy benchmarks -go run ./cmd/ate-setup delete benchmarks -``` - -`--skip-build` and `--worker-memory` are only available when invoking -`benchmarking/deploy_locust.sh` directly. - -### Settings - -`ate-setup` reads these settings in addition to those in -[docs/operator-install.md](../docs/operator-install.md#settings), through the same -flag, environment variable, and configuration document channels. - -| Key | Environment variable | Flag | Default | Applies to | -|--------------------------|-------------------------------|----------------------------|----------|------------------------------------------| -| `benchmark.actorMemory` | `BENCHMARK_ACTOR_MEMORY` | `--benchmark-actor-memory` | — | (global) | -| `benchmark.sandboxClass` | `ATE_BENCHMARK_SANDBOX_CLASS` | `--sandbox-class` | `gvisor` | `deploy benchmarks`, `delete benchmarks` | -| `benchmark.workerCount` | `ATE_BENCHMARK_WORKER_COUNT` | `--worker-count` | `1` | `deploy benchmarks`, `delete benchmarks` | - -## Running Tests - -### Locust Web UI -* Run `kubectl port-forward svc/locust -n benchmarking 8089:8089` -* Visit `http://localhost:8089` in your browser to configure and start the load test. - -The different user classes you can select are different types of load behaviors -you can throw at the system. Note that the "CounterUser" load type requires -that the counter demo be installed. - -You can also configure things like the number of users, how quickly those users -are spawned, the frequency with which requests are made and whether or not tracing is -enabled. - -User classes implemented in boomer rather than Python are selected at deploy -time — the stack runs one per deployment: - -```bash -./benchmarking/locust/deploy.sh --deploy --user-class durdir -``` - -### Headless (automation only) - -`runner.py` runs a test without the web UI, writing CSVs, logs and traces to -`--dest`. The nightly automation submits it as a Job on the test cluster; it is -not a local entry point. See [automation/README.md](automation/README.md). - -```bash -python3 runner.py -f tests/.py -t 1m -u 1 --name --dest /tmp/bench -``` - -Four flags control the optional post-run measurements described in -[Benchmark output files](#benchmark-output-files): - -* `--cluster-facts` / `--no-cluster-facts`: read node capacity and worker pod - count from the Kubernetes API once the run ends, to derive density frontiers, - and worker pod restarts and evictions around the run, for `oom_events`. - On by default. Pass `--no-cluster-facts` to skip Kubernetes API discovery. -* `--prometheus-url`: the Prometheus to harvest server-side telemetry from. - Defaults to the in-cluster service installed by - [Optional: Prometheus + Grafana](#optional-prometheus--grafana). -* `--atelet-lag-s`: how long to wait after the run before reading the - atelet's metrics. Defaults to 70. -* `--phase-breakdown` / `--no-phase-breakdown`: read the atelet and worker pod - logs once the run ends and add the suspend/resume phase percentiles to - `stats.jsonl`. On by default; needs `pods/log` access in `ate-system` and - `benchmark-workloads` (granted in `automation/manifests/runner-job.yaml.tmpl`). - See [analysis/README.md](analysis/README.md#automated-runs). - -Test-specific flags are appended to the same command; see the sections below. - -### DurDir Benchmark - -The DurDir benchmark evaluates actor suspend/resume performance, disk persistence overhead, -and state restoration latency when a durable directory is attached to the actor. - -#### DurDir Configuration Knobs - -* `--durdir-file-size-bytes`: Size in bytes of the data file (default `8388608` = 8 MiB). -* `--resume-mode`: Resume trigger mode: - * `explicit` (default): Client invokes the `ResumeActor` RPC before sending traffic. - * `implicit`: Client sends traffic through the router without an explicit wake RPC, testing traffic-triggered resume. -* `--durdir-read-mode`: Verification read mode: - * `data` (default): Server returns full payload bytes for client-side SHA-256 verification. - * `digest`: Server hashes the file and returns size and digest, reducing network transfer. -* `--durdir-template`: ActorTemplate name: - * `glutton-durdir-data` (default): Attaches a durable data directory without memory snapshot restore. - * `glutton-durdir-full`: Attaches a durable data directory and performs a full memory snapshot restore. - * `glutton-storage`: Attaches an external CSI volume instead of a durable - directory. `deploy_locust.sh` and `workloads/deploy.sh` deploy it only - when `--storage-class-name` names an existing StorageClass. To run it on - NFS, install with `go run ./cmd/ate-setup setup csi nfs` and pass - `--storage-class-name csi-nfs-sc`, the class the e2e tests also use. - -#### DurDir Reported Metrics - -* `DurDirWrite`: Initial truncate-write creating the data file. -* `DurDirServeInitial`: First read immediately following file creation. -* `SuspendActor`: Actor suspend latency (snapshot creation + persistence upload). -* `ResumeActor`: Actor resume latency. -* `DurDirServeAfterResume`: First read after resume (measures page faults / lazy load overhead on restored volume). -* `DurDirServeWarm`: Subsequent read within the same active cycle (cached state baseline). -* `DurDirOverwrite`: In-place file overwrite with checksum verification. - -### Sweperf Benchmark - -The sweperf benchmark replays a recorded SWE-Perf task inside an actor, suspending and -resuming between cycles to measure the cost of actor state transitions under a realistic -agent workload. One task is four cycles by default. Each task runs on a fresh actor: once -the last cycle finishes, the actor is deleted and the user creates a new one for the next -task. - -#### Sweperf Tasks - -Each task is one entry (name, digest-pinned image, step count) in -`workloads/manifests/sweperf/sweperf-images.json`. Its ActorTemplate, `sweperf-`, -is generated into `workloads/manifests/sweperf/generated/` and checked in. To add or change -a task, edit the catalog and regenerate: - -```bash -SWEPERF=benchmarking/workloads/manifests/sweperf -python3 -m venv "${SWEPERF}/venv" && "${SWEPERF}/venv/bin/pip" install -r "${SWEPERF}/requirements.txt" -"${SWEPERF}/venv/bin/python" "${SWEPERF}/generate_sweperf_templates.py" '*' -"${SWEPERF}/venv/bin/python" "${SWEPERF}/generate_sweperf_templates.py" --check # fail if generated/ is missing or stale -hack/verify/benchmarking-python.sh # unit tests, as CI runs them -``` - -Deploy a task and the sweperf user class: - -```bash -WORKLOAD_TEMPLATES=sweperf-django-11099 ./benchmarking/workloads/deploy.sh --deploy -./benchmarking/locust/deploy.sh --deploy --user-class sweperf -``` - -Then start the run with `--sweperf-template sweperf-django-11099` and -`--sweperf-total-steps 15`, set in the Locust web UI or through `LOCUST_SWEPERF_TEMPLATE` -and `LOCUST_SWEPERF_TOTAL_STEPS`. The client takes the step count from that option, not -from the template, so it must match the catalog. - -#### Sweperf Reported Metrics - -All rows are in milliseconds. CEL (command execution latency) is the time the trace commands -ran inside the sandbox, as reported by `replay.py`. - -* `ResumeToFirstExec`: Resume RPC start until the sandbox accepts the cycle's `/execute`. -* `CycleCEL`: CEL for one cycle. -* `TaskCEL`: CEL summed over one task. -* `TaskWallClock`: Client wall clock for one task, excluding inter-cycle think time and actor - startup. -* `CreateAtespace`, `CreateActor`, `ResumeActor`, `SuspendActor`, `DeleteActor`: Server-side - elapsed time for each control-plane RPC from the response trailer, or client time without one. -* `_rtt`: Client round trip for the RPC of the same name, recorded only when the trailer is - present, so network and queueing overhead stays visible separately. -* `Workload_Cycle_`: Client time for cycle `n`: `POST /execute` plus `/status` polling until - the job finishes. Polled every `--sweperf-poll-interval-ms` (default 100), so this row sits - up to one interval above the job's actual end. - -The liveness check on a new actor already has it running, so the first cycle of every task -runs on a freshly started actor and its resume is a no-op. Its successful `ResumeActor`, -`ResumeActor_rtt` and `ResumeToFirstExec` samples are not recorded; failures still are. - -The last cycle of a task is not suspended: the actor is deleted straight from running. A -task of `n` cycles therefore records `n - 1` `SuspendActor` samples, and `TaskWallClock` -does not include a final suspend. `DeleteActor` therefore times deleting a running actor, -including its sandbox teardown, so it is not comparable with older runs that suspended the -actor before deleting it. - -`locust_users` counts live actors rather than virtual users: it rises when a user starts an -actor and falls when that actor is deleted, so it dips between tasks while a user waits to -start its next actor. - -`CreateAtespace` is sent once per worker, not per task, and again only if `CreateActor` -reports the atespace missing. - -When the run ends, shutdown cancels cycles still in flight. Calls cut short this way record -nothing, and the interrupted task records no `TaskCEL` or `TaskWallClock` row, so the end of -a run does not show up as a burst of failures. - -#### Sweperf Log Lines - -The boomer log also carries per-task facts for post-processing such as the goodput script in -`tools/goodput`, which decides from them whether a task was good. The benchmark judges -nothing. Durations are in milliseconds. - -* `Completed sweperf cycle`: `actor`, `cycle`, `cel_ms` (0 when the execute failed), - `wall_ms` (resume start to cycle end), `ok` (the execute succeeded, and so did the suspend - if there was one), `resume_ms` (as on the `ResumeActor` row; omitted for a no-op resume) - and `ttfe_ms` (resume start to the asynchronous ack; omitted without one, and for a no-op - resume even when its execute got one). -* `ResumeActor failed in step`: also carries `wall_ms`. The cycle is retried on the next - iteration, and the failed attempt's time counts toward the task's `wall_ms`. -* `sweperf task record`, one per finished task: `v` (1), `user` (boomer goroutine ID, unique - per boomer process), `actor`, `template`, `cycles` and `steps` (the cycle and trace step - counts the task ran, resolved when it started, so a mid-run change to - `--sweperf-num-cycles` or `--sweperf-total-steps` applies from the next task; `cycles` is - capped at `steps`), `task_start` (when the session started) and `end`, `startup_ms` - (`CreateActor` until the sandbox answers `/status`), `cel_ms` and `wall_ms` (as in `TaskCEL` - and `TaskWallClock`), `think_ms` (think time between the task's cycles), `resume_max_ms` - (slowest non-no-op resume), `ttfe_max_ms` (slowest `ttfe_ms`; omitted when no cycle after - the no-op resume got an asynchronous ack, so always for a one-cycle task), `retries` - (failed resumes), `failed`, and the first failure's `fail_step` (`resume`, `execute` or - `suspend`) and `fail_cycle`. -* `sweperf on_start failed`: also carries `user`, to tie a failed start to its user. -* `sweperf shutdown started`: the worker began tearing down. Startups and cycles in flight - are canceled: interrupted tasks log no task record, and startup failures logged after it - belong to the teardown, not the run. - -### ate-env Benchmark - -The `ate-env` benchmark (`--user-class ate_env`) exercises the -[`ate-env-guest`](https://github.com/agent-substrate/env/tree/1c08a7a578afd896373511a83beb242aec1425c0/cmd/ate-env-guest) -sandbox image from `agent-substrate/env`. It follows `glutton`'s -wake/hold/hibernate lifecycle, but replaces synthetic HTTP `/ping` calls with a -real `"hello world"` process execution inside the guest (`StartProcess(["echo", -"hello world"])` + `StreamProcessOutput(follow: true)`) over gRPC directly -through `atenet-router` (`1 dns:///` gRPC channel per VU). - -Build the guest image from `agent-substrate/env` at commit `1c08a7a` (see -[`workloads/manifests/ate-env-template.yaml.tmpl`](workloads/manifests/ate-env-template.yaml.tmpl)), -then deploy the `ate-env` template and user class: - -```bash -ATE_ENV_IMAGE= WORKLOAD_TEMPLATES=ate-env ./benchmarking/workloads/deploy.sh --deploy -./benchmarking/locust/deploy.sh --deploy --user-class ate_env -``` - -#### ate-env Configuration Knobs - -Reuses `glutton`'s runtime and startup knobs: -`--min-wait-time`, `--max-wait-time`, `--min-live-time`, `--max-live-time`, -`--max-pings-per-wake`, `--lifecycle-mode` (`sync` | `async` | `none`), -`--trace-probability`, and `--actors-per-user`. - -#### ate-env Reported Metrics - -* `CreateAtespace`, `CreateActor`, `ResumeActorFirstResume`, `ResumeActor`, - `SuspendActor`, `SuspendActorAsync`, `DeleteActor`: Control-plane lifecycle - RPC latencies. -* `AteEnvExec`: Combined end-to-end latency of `"hello world"` (`StartProcess` + - `StreamProcessOutput` to exit with `exit_code == 0` and - `stdout == "hello world\n"`). -* `CrashCount`: Actors that entered `ACTOR_STATE_CRASHED` during resume. - -### Agent-Session Benchmark - -The agent-session benchmark (`--user-class agentsession`) emulates a fleet of -coding agents on Substrate. Each locust user is one session: an actor driven -through a scripted 20-step coding task ("clone a repo, build it, fix a test, -refactor, package it"), where every step costs the sandbox the CPU, memory, -disk, and network a real coding agent's action would. Between steps the agent -is "waiting for the LLM to think": the driver **suspends the actor** for the -step's think time, and the next step's first request **wakes it through the -atenet router** (request parking). Mostly-idle sessions plus fast wake is -exactly the oversubscription story this measures. - -The entire workload is a YAML script. The default, -[`internal/benchmarking/boomer/agentsession/scripts/coding-session.yaml`](../internal/benchmarking/boomer/agentsession/scripts/coding-session.yaml), -has one entry per step naming what the agent is doing and the resource ops -that act it out. To change the workload, edit it or add a sibling file and -select it with `--agentsession-script` (see [Writing a -script](#writing-an-agent-session-script)). Each session is an actor from -the stock `glutton` template; steps are sequences of glutton RPCs: - -| Step | The agent is… | Sandbox effect | -|---|---|---| -| 01_read_task | reading the task prompt | fill 32Mi RAM (agent context) | -| 02_clone_repo | `git clone` | 16Mi arrives over the network → disk; 0.5s CPU | -| 03_explore_tree | listing/grepping the tree | disk read (digest); 0.2s CPU | -| 04_read_key_files | opening files into context | disk read shipped back out; 8Mi RAM churn | -| 05_install_deps | `pip install` / `go mod download` | 32Mi over the network → disk; 1.5s CPU ×2 | -| 06_first_build | first full build | fill 64Mi RAM; 3s CPU ×2; 24Mi disk write | -| 07_run_unit_tests | running the test suite (one fails) | disk read; 2.5s CPU ×2 | -| 08_reason_about_failure | tracing the bug (long LLM turn, 8s think) | full RAM page-walk after the wake | -| 09_edit_source | applying the fix | 64Ki patch over the network; 4Mi disk write | -| 10_incremental_build | rebuilding changed packages | 1.2s CPU ×2; 8Mi disk write | -| 11_rerun_failed_test | re-running the failing test | 0.8s CPU | -| 12_write_new_tests | authoring regression tests (6s think) | 128Ki over the network; 2Mi disk write | -| 13_run_new_tests | running the new tests | disk read; 1s CPU | -| 14_full_test_suite | full-suite regression run | 4s CPU ×2; disk read; 8Mi RAM churn | -| 15_lint_format | lint + format pass | disk read; 0.9s CPU | -| 16_refactor | multi-file refactor (8s think) | 512Ki over the network; 6Mi disk write; 0.6s CPU | -| 17_rebuild | full rebuild | 32Mi RAM churn; 2s CPU ×2; 16Mi disk write | -| 18_final_test_suite | final full-suite run | 3.5s CPU ×2 | -| 19_package_artifact | building the release package | disk read; 1s CPU; 24Mi disk write | -| 20_commit_and_summarize | committing + summarizing | 256Ki disk write; 24Mi shipped back out; RAM walk | - -The script needs bigger actors than the 256Mi default: it holds ~96Mi of -RAM arrays + ~110Mi of tmpfs files, and the observed guest peak with -allocator transients is ~320Mi (512Mi OOMs). Deploy the workloads with -`--actor-memory 1Gi`. Every script declares that floor as -`min_actor_memory`; the worker reads the `glutton` template's memory limit -at start and refuses to run a script against a smaller actor, so a too-small -deployment fails loudly instead of showing up as OOM-flaky steps. - -```sh -./benchmarking/deploy_locust.sh --deploy --sandbox-class gvisor --actor-memory 1Gi -./benchmarking/locust/deploy.sh --deploy --user-class agentsession -``` - -#### Agent-Session Configuration Knobs - -* `--agentsession-script` — built-in script variant to run, by file name - under `internal/benchmarking/boomer/agentsession/scripts/` (default - `coding-session`). Resolved when a session starts, so a change takes - effect for sessions started after the next swarm; sessions already - running finish on the script they started with. -* `--agentsession-script-file` — path, on the boomer worker, of a script - YAML to run instead of a built-in variant; wins over - `--agentsession-script`. Normally set for you by - `locust/deploy.sh --agentsession-script FILE` (below). -* `--agentsession-think-scale` — multiplier on every think gap; 0.5 makes the - fleet twice as chatty, 4.0 models slow reasoning models (default 1.0). Each - gap gets ±20% jitter so sessions don't move in lockstep. -* `--resume-mode implicit|explicit` — implicit (default) lets the parked - first request wake the actor; explicit issues ResumeActor before traffic. -* `--lifecycle-mode sync|async|none` — synchronous suspend (`sync`, - uploads the snapshot before returning), asynchronous suspend (`async`, - default, uploads the snapshot in the background), or `none` to keep actors - resident in memory between steps without hibernating. -* `--agentsession-snapshot-steps` — comma-separated 1-based step numbers after - which the actor is hibernated (for example, `3,10,20,30,40,50,61`); between - all other steps the actor stays resident. -* `--agentsession-phase-in` — seconds over which session starts are spread: - each user waits a uniform random share of it before creating its actor - (default 0, every session starts at once). Locust's ramp spreads the fleet - over seconds and every session begins at step 1, so without it the - sessions reach each later step together; a step that dwells longer than - the ramp then asks every worker for more slots at once than it has. Set - it to a few think gaps for density runs. -* `--agentsession-resume-retry-budget` — seconds a step keeps retrying a - wake the fleet had no room for (ResumeActor `ResourceExhausted` or - `Unavailable`, router 503/504/429) with 50ms-to-1s backoff before giving - the step up until its next think gap (default 60; `0` gives the wake one - attempt). Each attempt is its own - `ResumeActor` or `WakeFirstTouch` sample; the whole wait lands in - `TimeToWake`. - -#### Writing an agent-session script - -A script is a named step list with a memory floor: - -```yaml -name: coding-session -min_actor_memory: 1Gi -steps: - - name: 01_read_task - agent: Boots, reads the task prompt, loads its context window - think: 2s - ops: - - fill_ram: {key: agent_context, size: 32Mi} - - ping: {} - - name: 02_clone_repo - agent: git clone of the target repository - think: 3s - ops: - - ingest: {key: repo_tarball, size: 16Mi} - - burn_cpu: {millis: 500, parallel: 1} -``` - -`think` is the suspended gap before the step (a Go duration, scaled by -`--agentsession-think-scale`); `name` keys the `Step_` stats row. -Sizes are Kubernetes quantities. Each op is one glutton RPC: - -| Op | Arguments | Sandbox effect | -|---|---|---| -| `ingest` | `key`, `size` | bytes cross the network from the driver and land in a file: a download | -| `burn_cpu` | `millis`, `parallel` (default 1) | spins `parallel` goroutines for `millis` of wall clock | -| `write_disk` | `key`, `size` | writes locally generated bytes to a file | -| `read_disk_digest` | `key` | reads and hashes a file; nothing ships back | -| `read_disk_data` | `key` | reads a file and returns its bytes: disk read plus egress | -| `fill_ram` | `key`, `size` | allocates a resident RAM array | -| `churn_ram` | `key`, `size` | re-randomizes part of an array in place, dirtying pages | -| `walk_ram` | `key` | touches one byte per page: demand-paging cost after a resume | -| `ping` | none | a minimal round trip through the router | -| `dwell` | `millis` | no request: the actor stays resident and idle, as a gateway does during a model round trip or a typing gap | - -Loading is strict: unknown op kinds or fields, an argument a kind does not -take, a read of a file nothing wrote, a walk of an array nothing filled, a -duplicate step name, or a `min_actor_memory` below the declared RAM plus -disk all fail before any actor is created. Built-in variants are checked by -`TestEmbeddedScriptsAreValid`, so a broken file cannot merge. - -#### Built-in script variants - -`coding-session` (default) is a coding agent working one task: 20 steps of -build, test and edit work, each preceded by a few seconds of LLM thinking -spent suspended; about 40 CPU-seconds of sandbox work per lap. - -`personal-assistant` is an always-on assistant of the OpenClaw or Hermes -Agent kind: one lap is one day in real time, with about 30 user messages -in 7 clusters, a heartbeat turn every 30 minutes while the user is awake, -hourly system-event crons, a morning briefing, model-catalog refreshes, -one context compaction and a nightly memory sweep (61 wakes, about 65 -CPU-seconds and 150 MiB written per lap, a 640Mi resident heap that grows -to about 960Mi). Each turn reads its session store, ships a 50 to 110 KiB -context out through the router, dwells for the model round trip, and -appends to its store, sized from an strace profile of OpenClaw; the -script's header comment lists the figures. Its think gaps are the real -idle gaps, so run it with a fractional think scale: -`--agentsession-script personal-assistant --agentsession-think-scale 0.02` -plays a day in about 30 minutes. Because every heartbeat and cron is a -wake, the script also shows what an always-on agent costs a system that -suspends it between events: an internal timer cannot fire in a suspended -sandbox, so each of those turns needs an external wake. - -`personal-assistant-light` is the same day with the RAM arrays scaled -down to 28Mi and everything else unchanged (files, reads, CPU, dwell and -the 61 wakes), for runs that want the always-on wake pattern with a small -memory image: about 40 to 95 MB per pause under gVisor, and under microVM -about 100 MB of guest before the script writes anything, bounded by the -actor's memory limit. It fits a 256Mi actor. Run it with the same -fractional think scale. - -To run a script of your own without rebuilding anything, hand it to the -locust deploy: - -```sh -./benchmarking/locust/deploy.sh --deploy --user-class agentsession --agentsession-script ./my-session.yaml -``` - -The script validates the file locally first (the same check the worker -runs, via `boomer-worker --check-agentsession-script`), uploads it as the -`agentsession-script` ConfigMap, mounts it into the boomer workers at -`/etc/agentsession/script.yaml`, and points the master's -`--agentsession-script-file` default at that path. Workers log the loaded -script's name, step count, and declared budgets on their first iteration. -To go back to a built-in variant, redeploy without the flag. - -#### Agent-Session Reported Metrics - -* `WakeFirstTouch`: latency of a dedicated ping sent before each step's ops — - in implicit mode that ping is what triggers the parked wake, so this row - **is** the user-visible wake latency, unpolluted by the step's own work. -* `TimeToWake`: the whole wake of a parked actor, from the first - `ResumeActor` attempt (in implicit mode, the first wake ping) to the ping - that found the actor serving, including every retry while the fleet had no - room. In implicit mode a first-attempt wake is the `WakeFirstTouch` sample; - in explicit mode it is `WakeFirstTouch` plus `ResumeActor`. A failed row is - a wake that failed: a step given up after - `--agentsession-resume-retry-budget`, or one that failed at once with an - error that is not a capacity error, such as a crashed actor or a router - 404 or 500. -* `Step_` (e.g. `Step_06_first_build`): wall time of that step's ops, - think gap excluded. -* `SuspendActor` / `ResumeActor` / `CreateActor` / `DeleteActor`: control-plane - lifecycle latencies. - -### Spawn Benchmark - -The Spawn benchmark creates a batch of actors once and measures how long each -actor takes from creation to its first answered ping, and how long the whole -batch takes. `tests.yaml` runs it as `spawn_smoke_10_actors` with -`shapes/spawn_shape.py`, which holds one user and ends the run once -`TimeToAllReady` is recorded. - -Each boomer process creates one batch; extra users in the same process do -nothing. Actors are named `spawn--` and deleted when boomer exits. - -#### Spawn Configuration Knobs - -* `--total-actors`: Actors in the batch (default `100`). -* `--spawn-concurrency`: Actors created concurrently (default `1`). -* `--actor-deadline`: Per-actor timeout in seconds, covering create, resume and - first ping (default `120`). - -The web UI shows the same fields; `0` keeps the value boomer-worker started with. - -#### Spawn Reported Metrics - -* `CreateActor`, `ResumeActor`, `GluttonPing`: Latency of each call. -* `ActorTimeToReady`: Per actor, from its first `CreateActor` attempt to its - first successful ping. -* `TimeToReady_pct` (`k` = 10, 20, … 100): From batch start until `k`% of - the batch was ready. -* `TimeToAllReady`: From batch start until the last ready actor answered. - Actors that failed show up as `ActorTimeToReady` failures instead. -* `CrashCount`: Actors that crashed during resume. - -The `actors_per_*` ratios in `trial_summary` are wrong for this test: they -count users × `--actors-per-user`, not `--total-actors`. - -### ResumePaused Benchmark - -The ResumePaused benchmark drives ate-api-server's cold resume path at a set -rate. A cold resume here is a ResumeActor of a suspended actor: ateapi takes the -actor lease, binds a worker, and restores the actor from its snapshot. A -ResumeActor of a running actor does none of that. Each user owns one actor -from the `glutton` template. On its first iteration the user creates the actor -and makes it cold with a resume and a suspend; every later iteration resumes the -actor and suspends it again, so every measured resume is cold. - -The warm-up runs in the step that added the user, unpaced, and is recorded -under its own names. - -It sends no requests to the actors themselves, so it runs on the fake data -plane, where it measures what a cold resume costs ate-api-server and Postgres -without the router or the restore in the way. The suspends run at the resume -rate, so a capacity it finds is one of resume and suspend cycles, not of resumes -alone. - -Actors are named `rp-` and deleted when boomer exits. If the atespace -cannot be ensured, boomer exits at startup. An actor that fails to warm is -deleted, and its user creates another on its next iteration. When a step lowers -the user count, the actors of the stopped users stay suspended, unused, until -boomer exits. - -To run the cold resume tests in `automation/tests.yaml` from a workstation, -see [automation/ateapi-cold-resume.md](automation/ateapi-cold-resume.md). - -#### ResumePaused Configuration Knobs - -* `--target-rps`: Cold resumes per second the whole boomer process paces to - (default `0`, unpaced). A `--ladder` step ending in `@target_rps`, as in - `200:3m@1000`, changes it as the step starts. Keep the users at or above the - rate times the p99 of one resume plus one suspend in seconds, or the step falls - short of its rate. - -The web UI shows the same field. - -Trim the start of each ladder step. A rate change reaches boomer at its next -poll of the master, within 10 s, and a step that adds users carries their -warm-up until they have all joined at the spawn rate. - -#### ResumePaused Reported Metrics - -* `ResumeActorCold`: Latency of each measured resume. A resume that finds its - actor already running fails, as no cold resume happened. -* `SuspendActorAsync`: Latency of the async SuspendActor that makes the actor - cold again. -* `SuspendActorAsyncRecovery`: After a failed cycle, the async SuspendActor that - makes the actor cold for the next one. It is load on the server, so it is - recorded, but apart from `SuspendActorAsync`. A recovery suspend that finds - the actor already suspended counts as a success. -* `CreateActor`, `ResumeActorWarmup`, `SuspendActorAsyncWarmup`: A user's - warm-up of its actor, in the step that added the user. - -#### Analyzing a ResumePaused Ladder Run - -`stats.csv` and the other rows of `stats.jsonl` cover a whole run, so they -blend every step of a ladder. After a ladder run the runner adds one -`stats.jsonl` row per step, with `metric` `ladder_step_001`, -`ladder_step_002`, and so on ([`locust/ladder_steps.py`](locust/ladder_steps.py)). -It reads each step's start and end from the run's log and queries the -benchmarking Prometheus over the step's steady window, from 30 s after the -step starts to 5 s before it ends (`window_start` and `window_end`). A row -holds the cold resume and pause rates, failures by operation -(`failures_`), client and server latency, the CPU and memory of -ate-api-server, Postgres and fake-atelet, and Postgres's connections, -commits and waits. - -The runner logs and skips a step with no end in the log and a step too short -to leave a window after the trim. A failed query adds no step rows at all -rather than a ladder with gaps, and leaves the run's other results as they -are. Read a step's figures against its `target_rps`: a `resume_rps` below it -means the step fell short of its rate. - -To pull the step rows out of a run's results: - -```bash -RUN=/runs//run_date=/run_ts=/run_tag= -gcloud storage cat "$RUN/stats.jsonl" | grep '"metric": "ladder_step_' -``` - -`` is the orchestrator's `--dest`, and its log has the full `RUN` path -on each `Uploaded ... -> gs://...` line. - -### Isolate Mode - -A real cold resume is limited by the node's restore capacity, which hides the -limits of ate-api-server and Postgres. Isolate mode takes the nodes out by -installing substrate with a fake data plane: - -```bash -go run ./cmd/ate-setup deploy ate-system --fake-data-plane -./benchmarking/workloads/deploy.sh --deploy --worker-count 200 \ - --fake-atelet-env FAKE_ATELET_DELAY=500ms -``` - -`deploy.sh` deploys the usual WorkerPool, but the install serves it with two -fakes from `cmd/benchmarking/isolate/` instead of worker pods: - -* `fake-workersync` stands in for ate-controller's WorkerPool controller and - worker syncer. It honors each pool's `spec.replicas` with fake Workers, none - backed by a pod, round robin on the nodes that run fake-atelet, and scales them - up and down with the pool, draining a Worker before deleting it. Each Worker - reports what a real worker would: cpu and memory from the pool's - `template.resources.limits`, or the node's allocatable without one, and 1000 - actors, ateworker's default, unless the pool's `ate.dev/fake-max-actors` - annotation says otherwise. It writes the pool's status, so - `kubectl get workerpool` and `kubectl scale workerpool` work. A node keeps - its Workers for `--node-grace`, five minutes by default, after its - fake-atelet pod goes, so a DaemonSet rollout does not move them; past that - they move to a node that still runs fake-atelet. -* `fake-atelet` takes atelet's place in its DaemonSet, on every node atelet - would run on, and answers every - lifecycle call with success after its `--delay`, without running or - saving any workload. It writes one placeholder object per snapshot, because - ate-api-server refuses to tag an empty one. Like ateworker, it mints the actor's - certificate from ate-api-server on every run and restore that names an - egress gateway, which the default install does; the mint is ate-api-server's - time, so it adds to `--delay`, the data plane's share of the call. - It also relays fake-workersync's capacity reports, as atelet relays ateworker's: - ate-api-server accepts a report only from the atelet on the Worker's node. - The relay admits only fake-workersync's pod identity. fake-atelet remembers - the epoch each Worker registers with and reports it from every run and - restore, as a worker process does; ate-api-server commits a resumed actor - only under its Worker's current epoch. It keeps the epochs in memory only, - and fake-workersync registers a Worker once, so a fake-atelet that restarts - after its Workers registered, in a DaemonSet rollout for example, fails - every resume on those Workers with Unavailable until they are replaced, - such as by redeploying the benchmark. - -The install also scales ate-controller to zero, whose worker syncer would -delete the fake Workers. Its other controllers stop with it: no NetworkPolicy -is made for a WorkerPool, and the egress MITM trust bundle is not derived. -Fake actors need neither. fake-atelet takes the storage settings of atelet's -objectstore-plugin sidecar, whose writes it takes over, so its snapshot -placeholders go to the storage the install points atelet at. The install -refuses a cluster whose WorkerPools still have worker pods, which would -outlive ate-controller. A normal `ate-setup deploy ate-system` -puts the real data plane back, and `delete ate-system` removes the fakes -either way. Never use the mode on a -cluster that serves real actors: actors placed on fake Workers do not exist. - -The installer only chooses the fake data plane; the benchmark tunes it. -fake-atelet reads each tuning flag from a `FAKE_ATELET_` variable in the -optional `fake-atelet-tuning` ConfigMap, which no install creates or resets: -`FAKE_ATELET_DELAY` sets `--delay`, `FAKE_ATELET_DELAY_RESTORE` overrides it -for Restore alone, and `FAKE_ATELET_MINT_ACTOR_CERTIFICATE=false` skips the -certificate mint. `deploy.sh --fake-atelet-env NAME=VALUE`, repeatable, writes -that ConfigMap to exactly the given entries and restarts fake-atelet when it -changes, before the WorkerPool is applied. - -Nothing answers behind the router, so the mode suits only a user class that -sends the actors no requests, such as `ate_api.py`, which creates, gets, -resumes and suspends actors. Every boomer user class pings its actors through -the router. In `tests.yaml`, an `isolate` block selects the mode, and its -keys become `deploy.sh --fake-atelet-env` entries: `delay` sets -`FAKE_ATELET_DELAY`, and so on. - -### Viewing Traces -You must have enabled otel tracing for your cluster to view traces. - -You can find trace IDs by viewing the `logs` tab in the Locust UI - -## Running at scale - -A dense run meets limits the demo path never reaches. The ones met so far, and -what to set before the run: - -* **Reaching the API and the router from outside the cluster.** The Locust - and boomer workers in this document run in the cluster and reach the - Services directly. A load generator outside it must not go through - `kubectl port-forward`: the forward resets under load and takes every - in-flight resume and pause down with it. Expose `svc/api` and - `svc/atenet-router` through a NodePort or LoadBalancer Service instead. - The port-forwards in this document are for the UIs only. -* **PostgreSQL connection slots.** Each ateapi replica opens up to - `pool_max_conns` connections in its read/write pool plus 5 for its owner - and watch pools. Unset, pgx sizes the pool at the node's CPU count (at - least 4), not the container's limit, so on a large node one replica can - take most of PostgreSQL's default `max_connections` of 100. Set it with - `--ateapi-postgres-pool-max-conns` on ate-setup and keep - `replicas x (pool max + 5)` under `max_connections` with slack for `psql` - and migrations (see - [connection sizing](../docs/dev/cloud-sql-scaling-guide.md#connection-sizing-for-a-target-throughput)). - To raise `max_connections`, add a `-c` argument to the StatefulSet in - `manifests/ate-install/postgres/postgres.yaml`, or install with - `--cluster-size size10`, which sets 300 and pins the pool at 64 but sizes - PostgreSQL for a dedicated node. When the slots run out, requests fail with - `INTERNAL` and `remaining connection slots are reserved`. -* **PostgreSQL probes.** Under load the probes miss their default 1 s - timeout. Three misses of the liveness probe (`pg_isready`) restart - postgres mid-run; three misses of the readiness probe (`nc`) drop the pod - from the Service, and ateapi loses its connections without a restart. Give - both `timeoutSeconds: 5` and `failureThreshold: 6`, and give the postgres - container CPU and memory requests sized for the run: on a saturated node - the request is its share of the CPU, and a starved postgres is what misses - the probes. -* **Parking budget.** `--parked-request-budget` (default `5s`) has to cover a - pause-and-resume cycle under load, tens of seconds, times the - oversubscription (live actors per worker slot); at 2x that is minutes. Watch - `atenet.router.parking.wait.duration{outcome="budget_exhausted"}`. See - [docs/request-parking.md](../docs/request-parking.md). -* **Synchronized bursts.** Clients that step in lockstep send one request per - actor at the same instant. Above the ext_proc circuit breaker Envoy answers - `503 ... reset reason: overflow` and the router never sees it. Jitter the - clients, or size `--extproc-max-requests`. -* **Kernel neighbor table.** Every sandbox has a link-local address, and - `net.ipv4.neigh.default.gc_thresh3` is global across network namespaces. - Above about a thousand sandboxes per host, raise the thresholds - (`gc_thresh1/2/3` = 4096/16384/32768). The symptom is an idle host with - `connection timed out` to its sandboxes and `arp_cache: neighbor table - overflow!` in `dmesg`. -* **Restores under full load.** A node's restore throughput falls as its - sandboxes burn CPU, and a restore wave onto a saturated node makes the - wakeup probe time out: the actor is crashed. Substrate does not steer - restores away from busy nodes, so keep CPU headroom instead: fewer resident - sandboxes per node, and admission (creates per second) paced to the restore - rate. -* **The kubelet.** Hundreds of sandboxes per node starve its heartbeat; after - the 300 s toleration the taint manager evicts every pod on the node. - Reserve CPU for it (`systemReserved`, `kubeReserved`) before going above a - few hundred sandboxes per node, and watch node `Ready`. - -Besides the Locust statistics, watch `ate.actor.crashes` (a crash is a finding, -not noise), `atenet.router.parking.*`, atelet restore and checkpoint p99, -`pg_stat_activity` against `max_connections`, node conditions, and `dmesg` for -OOM and neighbor-table lines. ateapi logs every RPC at INFO and its container -log rotates at 10 MiB, so read lifecycle events from the collector rather than -from `kubectl logs`. - -## Benchmark output files - -A run writes the following to `--dest`. Each run produces them fresh; none of -them are checked into the repository. - -* `status.json`: `locust_exit_code` and `stats_generated`. Deliberately just - those two keys, because it is what CI orchestration reads to decide whether a - trial ran at all. -* `stats.csv`, `stats_history.csv`, `failures.csv`, `exceptions.csv`: Locust's - own CSV output. `stats_history.csv` has a row per operation every second as - well as the `Aggregated` row, so each operation's latency can be charted - over the run; filter to `Aggregated` for run-wide numbers. -* `logs.txt`, `traces.txt`: the runner log, and the trace IDs seen during the run. -* `stats.jsonl`: one JSON object per line, one per metric. Every row carries - the same five keys: `timestamp`, `tag`, `test_name`, `metric`, and a flat - `measurements` map holding that metric's numbers. The `phase_*` rows and - the `phase_breakdown_summary` row are the suspend/resume phase percentiles - read from the node logs (see - [analysis/README.md](analysis/README.md#automated-runs)). -* `server_summary.json`: server-side telemetry harvested from Prometheus, - including the per-sample bin-packing timeseries. - -### Density frontiers - -With cluster discovery enabled, `stats.jsonl` gains a `trial_summary` row -describing how densely actors are packed onto the hardware. Its `measurements` -map holds the raw facts and the derived numbers side by side. - -* `machine_type`, `node_count`, `allocatable_cores`, `allocatable_ram_gb` - (GiB), `worker_pod_count`: the measured facts, before any arithmetic. - Capacity covers the nodes the worker pods are running on rather than the - whole cluster, so a separate infrastructure pool is not counted. They are - recorded so the ratios below can be re-derived later, or recomputed against - a different denominator. -* `kubernetes_version`: the worker nodes' Kubernetes version (kubelet version). -* `worker_actor_capacity`, `worker_memory_limit_gb` (GiB): the capacity each - worker pod declares, its ateworker `--max-actors` (1000 when unset) and its - container memory limit (null when unset, as the node bounds it instead). - `sandbox_class`: `gvisor` or `microvm`, from the worker image. A mixed pool - gives a sorted comma-joined list for each. An unreadable value is skipped - without affecting the others, and logged once. -* `multi_actor_worker`: `true` when `actors_per_pod_p50` is above 1. -* `run_start`, `run_end`: Unix seconds bracketing the run, for finding its - server-side metrics after the cluster is deleted. -* `metadata_`: one per key of the free-form `--metadata ''` object, - for facts only the caller knows. Non-string values are stored as JSON, and - null values are skipped. The orchestrator passes `cluster_name`, - `cluster_location` and `project_id` from the target cluster, plus any keys in - the `BENCHMARK_METADATA` environment variable (a JSON object) and in the - test's own `--metadata` flag, whose keys win on conflict. -* `actors_per_node`, `actors_per_vcpu`, `actors_per_gb_ram`: the most actors - Locust reported running, over the matching capacity. The `-u` flag only - stands in when no sample was read. -* `actors_per_pod_p50`, `actors_per_pod_p90`, `actors_per_pod_p99`: actors per - worker pod across the run. Reported as a distribution rather than one - average, and it spans ramp-up too, because a custom load shape has no - single user count to call steady. -* `aggregate_failure_ratio`: failures over requests for the run. -* `_failure_ratio`: the same ratio for every operation Locust - reported, so each test carries its own names through. The operation name is - lowercased with underscores, so `DurDirWrite` becomes - `dur_dir_write_failure_ratio`. A key is absent when the test has no such - row, and null when the row ran no requests. - -The six `actors_per_*` ratios rest on three assumptions. Read them before -comparing numbers across runs: - -* **Actors are derived, not counted.** Locust only sees virtual users, so the - numerator is the peak user count times `--actors-per-user`. No server-side - gauge counts resident actors: `ate.actor.stats.sampled_actors` drops any - actor without a live resource measurement, so suspended ones fall out. It - is recorded as `active_actors` in `server_summary.json`, a running-actor - count rather than a resident one. -* **The denominators are read once, after the run.** A cluster that autoscaled - mid-run is measured at its final size, so the ratio pairs a peak from one - moment with a capacity from another. The same holds for every worker and - node fact above, including `kubernetes_version` and the worker capacity, and - so for `multi_actor_worker`, which is derived from `actors_per_pod`. -* **The peak assumes every actor is alive at once.** A workload that creates - and deletes actors as it goes never holds them all at the same time, so its - real density is lower than reported. - -### Server ground truth - -With a reachable Prometheus, `server_summary.json` records what the server -actually did, independent of what the load generator reported. - -* `cluster_packing`: workers holding at least one actor (`partial` or - `at_capacity`) over the pool size at each sample (the sum of all - `ate_workerpool_workers` states), as a percentile `summary` plus the - per-sample `timeseries` it was computed from. -* `node_psi.cpu_stall_pct`, `mem_stall_pct`, `io_stall_pct`: kernel pressure - stall percentages on the nodes hosting a worker pod, with every node's - samples pooled. `node_psi.nodes` is how many nodes were seen. -* `pod_psi.cpu_stall_pct`, `mem_stall_pct`, `io_stall_pct`: the same for each - worker pod's cgroup slice, which also holds its gVisor sandbox, with every - pod's samples pooled. `pod_psi.pods` is how many pods were seen. -* `snapshots.size_p50_mb` through `size_p99_mb`: actor memory image sizes. -* `snapshots.size_avg_mb`: mean memory image size. -* `snapshots.restore_p50_s` through `restore_p99_s`, `restore_mean_s`, and the - same for `checkpoint_*`: how long the atelet took to resume or suspend an - actor (`ate_actor_{restore,checkpoint}_duration`, `total` phase). Every - snapshot kind counts, as in the sizes: the actor's own snapshot and a - template's first start from its golden snapshot. The buckets reach 60s, so a - slower operation reads as 60s. -* `snapshots.checkpoints_in_window`, `checkpoints_cumulative`: checkpoint - volume over the steady-state window, and since the atelet started. -* `snapshots.checkpoint_mb_s`: bytes written per second spent checkpointing, - not per second of wall clock. -* `active_actors`: running actors (`ate_actor_stats_sampled_actors`), as a - percentile `summary` of the cluster-wide count, `atelets` seen, and a - `timeseries` every 10s over the whole run (ramp-up included). Each point has - the cluster-wide count and `active_atelets` hosting an actor. As the - assumptions above note, the gauge drops actors without a live measurement, - so suspended ones fall out and the count dips while actors are suspended. - The atelet stops exporting when a node has no running actor, so at a step - where at least one atelet reported, a missing one counts as 0. A step where - none reported is skipped, not 0, since a failed scrape looks the same; this - mostly hits 1-user runs. If the metric never appeared during the run, every - field is `null`. `atelets` counts atelet processes seen in the run, not - nodes: the collector keeps an exited atelet's last value for about 5 - minutes, so a restart can briefly count twice, and an atelet scraped both - directly and through a collector counts twice. -* `working_set.node`, `worker`, `atelet`: memory working set in GiB from - cAdvisor on the nodes hosting a worker pod, per worker pod (every actor on it - included), and per atelet on those nodes. Each has a percentile `summary` - (plus `max`) over the steady-state window, `count` of series seen in it, and - a `timeseries` every 10s over the whole run with the sum (`total_gb`) and - largest (`max_gb`). A spike shorter than 10s can be missed. -* `working_set.actor`, `per_actor`: the same shape for the actors' own working - set as the atelet reports it (`ate_actor_stats_memory_working_set_bytes`), - per atelet with every template summed, and the average per measured actor - (`ate_actor_stats_sampled_actors`) on each atelet; `count` is atelet - processes, keyed like `active_actors`. As `per_actor` is an average per - atelet, its `max` is the highest atelet average, not the largest actor, and - its `total_gb` has no meaning. A gVisor actor is read from its sandbox cgroup - on the host, a microVM actor from inside the guest. As with `active_actors`, - suspended actors fall out, but through the telemetry-meter a series that - stops being exported keeps its last value for up to 5m. Missing samples are - skipped, not counted as 0. The atelet samples every - `--actor-stats-poll-interval` (default 1m), so these move in steps of a - minute or more. `actor` matches one worker pod's actors only when the node - runs one worker pod. -* `oom_events.node`, `pod`: OOM kills over the run, from cAdvisor's - `container_oom_events_total`. `node` counts only global OOMs. `pod` counts - kills inside a worker pod (actor sandboxes and the worker container) and is - best effort: a non-zero value is real, but a kill whose cgroup is gone before - the scrape is missed. A node-allocatable OOM (`oom_memcg=/kubepods.slice`) - isn't global, so it isn't in `node`, and `pod` usually misses it; the kubelet - evictions that often come with it show in `evicted`. OOMs inside a microVM - guest are invisible and show up as task failures. Memory PSI (`node_psi` / - `pod_psi`) is the early warning. -* `oom_events.worker_restarts`, `worker_oomkilled_pods`, `evicted`, - `worker_pods_lost`: from the Kubernetes API, read before and after the run. - Restarts on a pod deleted during the run are lost. `worker_oomkilled_pods` - counts pods whose last restart was `OOMKilled`, not kills; a group kill of - the whole worker (`memory.oom.group=1`) shows here. `evicted` is kubelet - eviction under node pressure (usually memory), not a kernel OOM kill. It also - counts replacement pods the kubelet refused to start while the node was under - pressure, as they get the same `Evicted` reason. It comes from `Evicted` - events, which last about 1h, so a run longer than that can undercount. - `worker_pods_lost` is pods gone by the end for any reason (eviction, - scale-down, recycling), not an OOM count; a pod created and deleted within - the run isn't counted. A node OOM can also show as a worker `OOMKilled` - restart, so never add the fields. A field is `null` when its Kubernetes read - fails, and all are with `--no-cluster-facts`. - -Every distribution reports p50, p90, p95 and p99 over the steady-state window. -The steady-state window runs from the first to the last Locust sample at 90% or -more of the run's peak user count, so ramp-up and teardown are left out. Under -a step-ladder load shape it covers only the top step. -Counts, means and `checkpoint_mb_s` come from the growth of the histogram's sum -and count across the window, and the size, restore and checkpoint percentiles -are interpolated inside its buckets. PSI is a one-minute rolling average read -every 10s. - -The atelet exports before Prometheus scrapes it, so the harvest waits -`--atelet-lag-s` seconds (default 70, enough for the OTel SDK's 60s default -export and a 10s scrape) and reads the snapshot, `active_actors` and actor -working set windows half that late, stamping `active_actors` and actor -working set points back by the same amount. The window ends at the last -full-load sample, so teardown suspends are left out. - -`metadata.start_ts` and `end_ts` bound the whole run, which the packing, -`active_actors` and `working_set` timeseries cover. `steady_start_ts` and -`steady_end_ts` bound the steady-state window. A flat subset of the numbers -also goes into a `server_summary` row in `stats.jsonl`. - -Neither the Kubernetes API nor Prometheus is required. If either is unreachable, -or discovery was skipped, the affected fields are written as `null` and the run -still succeeds. A `null` means the value was not measured. It never means zero. - -## Suspend/resume phase breakdown - -`SuspendActor` and `ResumeActor` latency can be attributed to the phases -inside them (object-storage transfer, hypervisor snapshot/restore, rootfs -assembly, ...) from the structured log records atelet and ateworker-microvm -emit. `analysis/collect_logs.sh` dumps the node logs of a run and -`analysis/phase_report.py` aggregates them into per-phase percentiles and -waterfalls of the slowest operations. See -[analysis/README.md](analysis/README.md). - -## Optional: Prometheus + Grafana - -Locust provides graphs, statistics, etc. via the UI. However, you -can install Prometheus/Grafana if you want richer details or -the ability to perform deeper analysis. Skip this section if -you're only using the Locust web UI. - -```bash -kubectl apply -f benchmarking/monitoring.yaml -``` - -Once installed: - -* Run `kubectl port-forward svc/grafana -n benchmarking 3000:3000` -* Visit `http://localhost:3000` in your browser. - -## Development - -### Generating gRPC Python clients - -The clients are not checked in. The locust and nighthawk-ingress images -generate them at build time, and `hack/verify/python-protos.sh` compiles them -on every PR, so a proto change needs no extra step. For local use, such as -editor completion, run `benchmarking/locust/codegen/generate.sh`. It manages -its own virtual environment under `locust/codegen/venv`. - -### Unit tests - -`locust/unit_tests` covers the runner's helpers and needs no cluster. From the -repository root: - -```bash -python3 -m unittest discover -s benchmarking/locust/unit_tests -``` - -`analysis/test_phase_report.py` covers the phase-log report's parser and -aggregation: - -```bash -python3 -m unittest discover -s benchmarking/analysis -``` +Moved to [docs/guides/benchmarking.md](../docs/guides/benchmarking.md). diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 30a9dfe83c..ad54f51d67 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -27,7 +27,7 @@ To run the cold resume tests from your workstation instead, see images via `ko` as part of their deploy steps — there's no separate image build step). The install sets `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` to the `ate-client` and `benchmarking/benchmark-runner` ServiceAccounts so the - runner's calls pass [authorization](../../docs/authorization.md), and sets + runner's calls pass [authorization](../../docs/guides/authorization.md), and sets `ATE_INGRESS_ALLOWED_SPIFFE_IDS` to `benchmarking/benchmark-runner` for the Envoy ingress router. For a `microvm` test the orchestrator also runs `hack/install-microvm-deps.sh --install` between the two, which diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 8d090d24d6..62ddeb12f1 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -13,7 +13,7 @@ go run ./cmd/ate-setup [global flags] [flags] These configure every command, and may appear before or after it, except `--version`, which takes no command. -[`docs/operator-install.md`](../../docs/operator-install.md#settings) lists +[`docs/guides/operator-install.md`](../../docs/guides/operator-install.md#settings) lists every setting with its environment variable. | Flag | Notes | @@ -29,7 +29,7 @@ every setting with its environment variable. | `--ingress-allowed-spiffe-ids IDS` | Comma-separated SPIFFE IDs (exact match) of the clients the ingress router lets reach actors; any listed client may reach any actor. Unset, no client may reach actors through the router. Both dataplanes require a Pod Identity CA client certificate on HTTPS and TLS CONNECT. The SPIFFE URI SAN allowlist may be checked during or after TLS, including after CONNECT acceptance, but unlisted clients must be denied before actor routing. The podidentity signer issues `spiffe://cluster.local/ns//sa/`. `ATE_INGRESS_ALLOWED_SPIFFE_IDS` when the flag is absent | | `--egress-gateway-extra-spiffe-ids IDS` | Comma-separated SPIFFE IDs that hold ate-api-server's `egress_gateway` role next to atenet-egress, such as a credential provider that mints actor JWTs. ate-setup passes atenet-egress's ID and then each listed ID as repeated `--egress-gateway-spiffe-id` flags; unset passes none, leaving ate-api-server's default. `ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS` when the flag is absent | | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | -| `--credential-provider JSON` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/egress-credential-injection.md`](../../docs/egress-credential-injection.md). `ATE_CREDENTIAL_PROVIDER` when the flag is absent | +| `--credential-provider JSON` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/guides/egress-credential-injection.md`](../../docs/guides/egress-credential-injection.md). `ATE_CREDENTIAL_PROVIDER` when the flag is absent | | `--ateapi-log-level LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). The server picks up a change without restarting. `ATE_API_LOG_LEVEL` when the flag is absent | | `--otlp-endpoint URL` | Send control plane telemetry to `URL` instead of the cluster default (see [`benchmarking/telemetry/README.md`](../../benchmarking/telemetry/README.md)). `ATE_OTLP_ENDPOINT` when the flag is absent | | `--preview GATES` | Preview gates passed as `--preview` to every substrate component: comma-separated gate names, or `ALL` for all. `ATE_PREVIEW` or `config.preview` when the flag is absent. An explicit `--preview=''` turns them all off, even when `ATE_PREVIEW` is set | diff --git a/cmd/atenet/README.md b/cmd/atenet/README.md index 3abe9e1a0e..afbc58a6ee 100644 --- a/cmd/atenet/README.md +++ b/cmd/atenet/README.md @@ -39,7 +39,7 @@ likely be split in the future for better scalability.) requests finish instead of resetting. The whole sequence must fit within `terminationGracePeriodSeconds` (see the manifest comments). An upgrade rolls the router as an ordinary Deployment update (see - [docs/upgrade.md](../../docs/upgrade.md)). The drain lets each old pod's + [docs/guides/upgrade.md](../../docs/guides/upgrade.md)). The drain lets each old pod's requests finish, except one still open about 15 seconds after the drain starts, which is cut. diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index 06777c6db6..3150638ef0 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -15,7 +15,7 @@ Router has several responsibilities: * Parks requests whose actor cannot be served immediately due to transient worker-pool saturation, retrying the resume until the actor is routable or a bounded wait elapses, instead of failing fast. See - [docs/request-parking.md](../../../../docs/request-parking.md). + [docs/concepts/request-parking.md](../../../../docs/concepts/request-parking.md). * Drains gracefully on SIGTERM: flips `/readyz` so the Service stops sending new connections, waits out endpoint propagation (`--drain-delay`), drains the dataplane's established connections (Envoy only — driven over its admin API; @@ -105,7 +105,7 @@ rule, `egress_passthrough` under a `tls_passthrough` rule, and `denied`, which matches no chain and closes the connection, when neither covers the SNI. The passthrough chain resolves the SNI and relays the bytes to it on the dialed port; the address the actor dialed plays no part (see -`docs/network-egress.md`). +`docs/guides/network-egress.md`). Identity on the request legs is `dev.ate.actor.identity`, the actor's SPIFFE ID that the outer chain set from the verified peer certificate and shares with @@ -125,7 +125,7 @@ deleted policy becomes a deny. Every policy denial answers a fixed Credential injection (`replace_headers`) runs on both request legs: a header the request carries is replaced with the credential from the provider, and a request without it is forwarded unchanged. See -`docs/egress-credential-injection.md`. +`docs/guides/egress-credential-injection.md`. ## adding a dataplane attribute diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index 2c86821b69..6439371119 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -1,363 +1 @@ -# `kubectl-ate` - -A Kubernetes-native CLI plugin for managing Substrate Actor and Worker lifecycles. - -## Running the CLI - -There are two ways to run the tool, depending on whether you are developing locally or installing it permanently. - -### 1. Install as a native `kubectl` Plugin -You can use `go install` to compile the tool and place the binary directly into your Go bin directory (which should be in your `$PATH`). Because the source folder is named `kubectl-ate`, Kubernetes will automatically recognize the resulting binary! - -```bash -go install ./cmd/kubectl-ate -``` -You can now run it seamlessly anywhere as a native Kubernetes command: `kubectl ate `. - -### 2. Run directly from source (Development) -If you are testing changes to the codebase, you can bypass compilation and run the CLI directly from the source tree: - -```bash -go run ./cmd/kubectl-ate -``` - -## Connection & Auto Port-Forwarding -By default, `kubectl-ate` will automatically read your `~/.kube/config`, discover the `ate-api-server` pods in your cluster, and establish a temporary background port-forward tunnel to execute gRPC calls securely. - -If you prefer to route traffic directly (e.g., through a LoadBalancer or when running natively inside a cluster pod), simply provide the `--endpoint` flag to bypass the tunnel. - -## Tracing -The CLI supports on-demand tracing using the `--trace` flag. When enabled, the CLI will generate a trace ID and signal to the server that it wants the request to be traced. - -**Prerequisites:** - -1. The Google Cloud project must have the **Cloud Trace API** enabled. You can enable it using: -```bash -gcloud services enable cloudtrace.googleapis.com --project=PROJECT_ID -``` - -2. The GKE cluster must have **Managed OpenTelemetry** enabled. Clusters created by -`setup-gcp create cluster` or `setup-gcp bootstrap` always enable it. For a cluster that does -not have it, enable it with: - -```bash -gcloud beta container clusters update CLUSTER_NAME \ - --project=PROJECT_ID \ - --managed-otel-scope=COLLECTION_AND_INSTRUMENTATION_COMPONENTS \ - --location=LOCATION -``` - -**Local (kind):** - -The kind overlay installed by `ate-setup --kind deploy ate-system` already provisions an in-cluster OpenTelemetry Collector and Jaeger in the `otel-system` namespace. No additional setup is required. - -Port-forward the Jaeger UI and invoke any command with `--trace`: -```bash -kubectl port-forward -n otel-system svc/jaeger 16686:16686 & -kubectl ate get actor my-counter-1 --trace -# open http://localhost:16686 and search for the most recent trace -``` - -## Global Flags -These flags can be appended to any command: - -| Flag | Short | Description | Default | -|---|---|---|---| -| `--kubeconfig` | | Path to your kubeconfig file | `~/.kube/config` | -| `--context` | | Name of the kubeconfig context to use | current context | -| `--endpoint` | | Manual gRPC endpoint override (e.g., `localhost:8080`) | | -| `--token-file` | | Path to a bearer token for ate-api authentication, or `-` for stdin | Kubernetes ServiceAccount token | -| `--output` | `-o` | Output format (`table`, `json`, `yaml`) | `table` | -| `--trace` | | Enable on-demand tracing for the request | `false` | -| `--verbose` | `-v` | Increase output verbosity; may be repeated or stacked (e.g., `-v`, `-vv`, `-vvvvv`) | `0` | - -Pass `--version` (`kubectl ate --version`) to print the CLI version and exit. - ---- - -## Command Reference & Examples - -### Getting Resources -List and inspect the state of actors and workers across the cluster. - -```bash -# List actors in one atespace; -a is shorthand for --atespace -kubectl ate get actors --atespace -kubectl ate get actors -a - -# List actors across all atespaces -kubectl ate get actors -A - -# Get a specific actor by name and output as raw YAML -kubectl ate get actor --atespace -o yaml - -# List all physical workers and see which actors are assigned to them -kubectl ate get workers - -# Filter workers by Kubernetes namespace, assigned-actor atespace, or -# worker pool labels (same flags as `top workers`) -kubectl ate get workers -n -kubectl ate get workers -a -kubectl ate get workers -l -``` - -> **Note:** `get actors` requires either `--atespace ` / `-a ` (one atespace) or `-A`/`--all-atespaces` (all atespaces) — there is no default atespace. Getting a single actor always requires `--atespace`/`-a`, since an actor is addressed by `(atespace, name)`. `-a` (lower-case) scopes to one atespace; `-A` (upper-case) spans all. - -> **Note:** Actors, workers, and actor templates are not Kubernetes CRDs — they live in the Substrate control plane's PostgreSQL database, not `etcd`. `kubectl get actor`, `kubectl get worker`, and `kubectl get actortemplate` will not return anything; only `kubectl ate get …` queries the control plane (see [Actor Templates](#actor-templates)). `kubectl get workerpool` *does* work, because pools are CRDs. - -#### `kubectl ate get actor` output columns - -| Column | Meaning | -|---|---| -| `ATESPACE` | The atespace the actor belongs to. Part of the actor's identity; folded into the storage key as `actor::`. | -| `NAME` | The actor's name. User-provided for application actors; UUID for the golden actor that each template materialises while building its golden tag. | -| `TEMPLATE` | The `ActorTemplate` the actor references, displayed as `/`. | -| `STATE` | Current lifecycle state (`ACTOR_STATE_RESUMING`, `ACTOR_STATE_RUNNING`, `ACTOR_STATE_SUSPENDING`, `ACTOR_STATE_SUSPENDED`, `ACTOR_STATE_CRASHED`, `ACTOR_STATE_DELETING`, `ACTOR_STATE_REVERTING`). | -| `WORKER POD` | The worker pod (namespace/name) currently hosting the actor. Empty while suspended. | -| `WORKER IP` | The pod IP of that worker. Empty while suspended. | -| `VERSION` | Monotonic integer that increments on every state transition (resume / suspend / checkpoint). Useful for distinguishing snapshots. | -| `AGE` | Time elapsed since the actor was created. | - -#### `kubectl ate get worker` output columns - -| Column | Meaning | -|---|---| -| `NAMESPACE` | The `WorkerPool` namespace. | -| `POOL` | The `WorkerPool` name. | -| `POD` | The worker pod name. | -| `STATUS` | `FREE` (idle, ready to receive an actor) or `ASSIGNED` (currently hosting an actor). | -| `ASSIGNED ACTOR` | If `STATUS=ASSIGNED`, the actor reference `/