From e329970a405d569531434e0dc721e9826994e680 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Wed, 28 Oct 2026 09:37:56 -0700 Subject: [PATCH 1/4] apierror: add WithReason and the ErrorInfo detail In order to tell the actor state that clears on its own, SUSPENDING or REVERTING, from those that do not, such as CRASHED, introduce a Reason to the apiError, ACTOR_SUSPENDING or ACTOR_REVERTING. FromError sends it as an ErrorInfo detail under the ate.dev domain. First of four commits that make the router fail fast on a crashed actor. Signed-off-by: Yuan Gao --- internal/apierror/apierror.go | 39 +++++++++- internal/apierror/apierror_test.go | 76 +++++++++++++++++++ .../ateinterceptors/ateinterceptors_test.go | 13 ++++ 3 files changed, 127 insertions(+), 1 deletion(-) diff --git a/internal/apierror/apierror.go b/internal/apierror/apierror.go index df0ef318fb..6fd848073e 100644 --- a/internal/apierror/apierror.go +++ b/internal/apierror/apierror.go @@ -31,13 +31,31 @@ import ( "errors" "fmt" + "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) +// ErrorInfoDomain is the domain of every ErrorInfo detail a substrate server +// attaches to a status. +const ErrorInfoDomain = "ate.dev" + +// Reason is the reason field of the ErrorInfo detail a FailedPrecondition +// carries. Each value names a transient actor state that a concurrent +// operation moves the actor out of, so a caller that fails fast on +// FailedPrecondition may retry these. +type Reason string + +const ( + ReasonActorSuspending Reason = "ACTOR_SUSPENDING" // the suspend in flight ends in SUSPENDED + ReasonActorReverting Reason = "ACTOR_REVERTING" // the revert in flight ends in SUSPENDED +) + type apiError struct { code codes.Code err error + // reason, when set, is sent as an ErrorInfo detail of the status. + reason Reason } func (e *apiError) Error() string { return e.err.Error() } @@ -70,6 +88,17 @@ func FailedPrecondition(format string, args ...any) error { return newf(codes.FailedPrecondition, format, args...) } +// WithReason returns err with reason attached, so FromError sends it to the +// client as an ErrorInfo detail. err must come from a constructor in this +// package; anything else maps to Internal, since the reason would be lost. +func WithReason(err error, reason Reason) error { + var e *apiError + if !errors.As(err, &e) { + return newf(codes.Internal, "WithReason on %v, which is not an apierror", err) + } + return &apiError{code: e.code, err: e.err, reason: reason} +} + // Aborted reports a request that lost a concurrency conflict, such as a // version mismatch. Retrying from a fresh read can succeed. func Aborted(format string, args ...any) error { @@ -135,7 +164,15 @@ func FromError(err error) (s *status.Status, ok bool) { } var apiErr *apiError if errors.As(err, &apiErr) { - return status.New(apiErr.code, apiErr.Error()), true + st := status.New(apiErr.code, apiErr.Error()) + if apiErr.reason == "" { + return st, true + } + withReason, err := st.WithDetails(&errdetails.ErrorInfo{Domain: ErrorInfoDomain, Reason: string(apiErr.reason)}) + if err != nil { + return status.New(codes.Internal, fmt.Sprintf("while attaching ErrorInfo reason %q: %v", apiErr.reason, err)), true + } + return withReason, true } if s := status.FromContextError(err); s.Code() != codes.Unknown { return s, true diff --git a/internal/apierror/apierror_test.go b/internal/apierror/apierror_test.go index 0cea41d7d2..d4b1a731be 100644 --- a/internal/apierror/apierror_test.go +++ b/internal/apierror/apierror_test.go @@ -20,6 +20,7 @@ import ( "fmt" "testing" + "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -34,6 +35,7 @@ func TestConstructors(t *testing.T) { {name: "NotFound", err: NotFound("tier %d", 1), wantCode: codes.NotFound}, {name: "AlreadyExists", err: AlreadyExists("tier %d", 1), wantCode: codes.AlreadyExists}, {name: "FailedPrecondition", err: FailedPrecondition("tier %d", 1), wantCode: codes.FailedPrecondition}, + {name: "WithReason", err: WithReason(FailedPrecondition("tier %d", 1), ReasonActorSuspending), wantCode: codes.FailedPrecondition}, {name: "Aborted", err: Aborted("tier %d", 1), wantCode: codes.Aborted}, {name: "PermissionDenied", err: PermissionDenied("tier %d", 1), wantCode: codes.PermissionDenied}, {name: "Unauthenticated", err: Unauthenticated("tier %d", 1), wantCode: codes.Unauthenticated}, @@ -188,6 +190,80 @@ func TestFromError(t *testing.T) { } } +// TestFromErrorReason checks the ErrorInfo detail a reason becomes, and that +// errors without one carry no detail. +func TestFromErrorReason(t *testing.T) { + tests := []struct { + name string + err error + wantReason Reason + }{ + { + name: "reason becomes an ErrorInfo detail", + err: WithReason(FailedPrecondition("actor is SUSPENDING"), ReasonActorSuspending), + wantReason: ReasonActorSuspending, + }, + { + name: "wrapped error keeps the detail", + err: fmt.Errorf("while resuming: %w", WithReason(FailedPrecondition("actor is REVERTING"), ReasonActorReverting)), + wantReason: ReasonActorReverting, + }, + { + name: "plain FailedPrecondition has no detail", + err: FailedPrecondition("actor is CRASHED"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, _ := FromError(tt.err) + var info *errdetails.ErrorInfo + for _, d := range s.Details() { + if i, ok := d.(*errdetails.ErrorInfo); ok { + info = i + } + } + if tt.wantReason == "" { + if info != nil { + t.Fatalf("FromError() ErrorInfo = %v, want none", info) + } + return + } + if info == nil { + t.Fatalf("FromError() has no ErrorInfo detail, want reason %q", tt.wantReason) + } + if info.GetDomain() != ErrorInfoDomain || Reason(info.GetReason()) != tt.wantReason { + t.Errorf("FromError() ErrorInfo = %s/%s, want %s/%s", info.GetDomain(), info.GetReason(), ErrorInfoDomain, tt.wantReason) + } + }) + } +} + +// TestWithReason checks that WithReason keeps the code of an apierror, also +// through a wrapping error, and maps anything else to Internal rather than +// dropping the reason. +func TestWithReason(t *testing.T) { + tests := []struct { + name string + err error + wantCode codes.Code + wantMsg string + }{ + {name: "apierror keeps its code", err: WithReason(FailedPrecondition("actor is SUSPENDING"), ReasonActorSuspending), wantCode: codes.FailedPrecondition, wantMsg: "actor is SUSPENDING"}, + {name: "wrapped apierror keeps its code", err: WithReason(fmt.Errorf("while resuming: %w", FailedPrecondition("actor is SUSPENDING")), ReasonActorSuspending), wantCode: codes.FailedPrecondition, wantMsg: "actor is SUSPENDING"}, + {name: "plain error maps to Internal and says why", err: WithReason(errors.New("boom"), ReasonActorSuspending), wantCode: codes.Internal, wantMsg: "WithReason on boom, which is not an apierror"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Code(tt.err); got != tt.wantCode { + t.Errorf("Code() = %v, want %v (err=%v)", got, tt.wantCode, tt.err) + } + if got := tt.err.Error(); got != tt.wantMsg { + t.Errorf("Error() = %q, want %q", got, tt.wantMsg) + } + }) + } +} + func TestFromErrorNil(t *testing.T) { s, ok := FromError(nil) if s != nil || !ok { diff --git a/internal/ateinterceptors/ateinterceptors_test.go b/internal/ateinterceptors/ateinterceptors_test.go index 07cc9bf98f..a6a26a3651 100644 --- a/internal/ateinterceptors/ateinterceptors_test.go +++ b/internal/ateinterceptors/ateinterceptors_test.go @@ -43,6 +43,7 @@ func TestStatusErrorInterceptor(t *testing.T) { handlerErr error wantCode codes.Code wantMsg string + wantReason apierror.Reason // ErrorInfo reason the caller must see, if any expectResponse bool }{ { @@ -80,6 +81,13 @@ func TestStatusErrorInterceptor(t *testing.T) { wantCode: codes.DeadlineExceeded, wantMsg: "while loading actor: context deadline exceeded", }, + { + name: "APIErrorReasonReachesTheCaller", + handlerErr: fmt.Errorf("while resuming: %w", apierror.WithReason(apierror.FailedPrecondition("actor is SUSPENDING"), apierror.ReasonActorSuspending)), + wantCode: codes.FailedPrecondition, + wantMsg: "actor is SUSPENDING", + wantReason: apierror.ReasonActorSuspending, + }, } for _, tt := range tests { @@ -116,6 +124,11 @@ func TestStatusErrorInterceptor(t *testing.T) { if st.Message() != tt.wantMsg { t.Errorf("expected message %q, got %q", tt.wantMsg, st.Message()) } + + info := errorInfoOf(t, err) + if got := apierror.Reason(info.GetReason()); got != tt.wantReason { + t.Errorf("ErrorInfo reason = %q, want %q", got, tt.wantReason) + } }) } } From c3a3b22e65c7ac9f526d5e656650c9ac27ba31d9 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Wed, 28 Oct 2026 09:37:56 -0700 Subject: [PATCH 2/4] ateapi: mark a resume during SUSPENDING or REVERTING with a reason Before, ensureWorkerAssigned refuses every state but SUSPENDED and RESUMING with one plain FailedPrecondition. The SUSPENDING and REVERTING arms now attach a reason through assignWorkerPrerequisiteNotMetError so that they differentiate from other states like CRASHED. Signed-off-by: Yuan Gao --- .../internal/controlapi/workflow_resume.go | 21 +++++++- .../controlapi/workflow_resume_test.go | 48 +++++++++++++++++++ docs/reference/api-guide.md | 1 + 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 5ae65943ce..cb961fd98e 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -231,6 +231,25 @@ func (w *ActorWorkflow) ensureVolumesCreated(ctx context.Context, actorRef resou return storedActor, nil } +// assignWorkerPrerequisiteNotMetError is the FailedPrecondition for an actor +// in a state ensureWorkerAssigned cannot start from. SUSPENDING and REVERTING +// carry an ErrorInfo reason, since a concurrent operation moves the actor out +// of them and a caller that fails fast on FailedPrecondition may retry. +func assignWorkerPrerequisiteNotMetError(actorRef resources.ActorRef, actor *ateapi.Actor) error { + const format = "AssignWorker prerequisite not met for Actor: %s (got: %v, want %s)" + state := actor.GetStatus().GetState() + var reason apierror.Reason + switch state { + case ateapi.ActorState_ACTOR_STATE_SUSPENDING: + reason = apierror.ReasonActorSuspending + case ateapi.ActorState_ACTOR_STATE_REVERTING: + reason = apierror.ReasonActorReverting + default: + return apierror.FailedPrecondition(format, actorRef, state, ateapi.ActorState_ACTOR_STATE_SUSPENDED) + } + return apierror.WithReason(apierror.FailedPrecondition(format, actorRef, state, ateapi.ActorState_ACTOR_STATE_SUSPENDED), reason) +} + // ensureWorkerAssigned assigns a worker and leaves the actor RESUMING. // Re-entry validates the recorded assignment before reuse. // Version conflicts re-read the actor before retrying. @@ -248,7 +267,7 @@ func (w *ActorWorkflow) ensureWorkerAssigned(ctx context.Context, actorRef resou return actor, worker, nil case ateapi.ActorState_ACTOR_STATE_SUSPENDED: default: - return nil, nil, apierror.FailedPrecondition("AssignWorker prerequisite not met for Actor: %s (got: %v, want %s)", actorRef, actor.GetStatus().GetState(), ateapi.ActorState_ACTOR_STATE_SUSPENDED) + return nil, nil, assignWorkerPrerequisiteNotMetError(actorRef, actor) } var assignedActor *ateapi.Actor diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 894e632b9e..60150d74a4 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -41,6 +41,7 @@ import ( "go.ate.dev/substrate/internal/volume" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" + "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" @@ -2233,6 +2234,53 @@ func TestEnsureVolumesCreated(t *testing.T) { } } +// ensureWorkerAssigned refuses every state but SUSPENDED and RESUMING with +// FailedPrecondition, and attaches an ErrorInfo reason only for the states a +// concurrent operation moves the actor out of, so a caller that fails fast +// on FailedPrecondition may retry those. +func TestEnsureWorkerAssigned(t *testing.T) { + t.Parallel() + tests := []struct { + name string + state ateapi.ActorState + wantReason apierror.Reason + }{ + {"suspending actor carries ACTOR_SUSPENDING", ateapi.ActorState_ACTOR_STATE_SUSPENDING, apierror.ReasonActorSuspending}, + {"reverting actor carries ACTOR_REVERTING", ateapi.ActorState_ACTOR_STATE_REVERTING, apierror.ReasonActorReverting}, + {"crashed actor carries no reason", ateapi.ActorState_ACTOR_STATE_CRASHED, ""}, + {"deleting actor carries no reason", ateapi.ActorState_ACTOR_STATE_DELETING, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + actor := &ateapi.Actor{ + Metadata: &ateapi.ResourceMetadata{Atespace: "team-a", Name: "id1"}, + Status: &ateapi.ActorStatus{State: tt.state}, + } + // The state switch runs before any store access. + w := &ActorWorkflow{} + _, _, err := w.ensureWorkerAssigned(context.Background(), actorRef, actor, nil) + if code := apierror.Code(err); code != codes.FailedPrecondition { + t.Fatalf("ensureWorkerAssigned code = %v, want FailedPrecondition; error = %v", code, err) + } + if !strings.Contains(err.Error(), "AssignWorker prerequisite not met") { + t.Errorf("ensureWorkerAssigned error = %v, want the AssignWorker prerequisite message", err) + } + st, _ := apierror.FromError(err) + var gotReason apierror.Reason + for _, d := range st.Details() { + if info, ok := d.(*errdetails.ErrorInfo); ok && info.GetDomain() == apierror.ErrorInfoDomain { + gotReason = apierror.Reason(info.GetReason()) + } + } + if gotReason != tt.wantReason { + t.Errorf("ErrorInfo reason = %q, want %q", gotReason, tt.wantReason) + } + }) + } +} + // filledUpScheduler hands out workers in order and refuses room on the ones // named in full. It stands in for a stale worker cache: Schedule offers a // worker that looked free, and the admit callback that BindActorToWorker runs diff --git a/docs/reference/api-guide.md b/docs/reference/api-guide.md index 68e883c867..4f61f2603f 100644 --- a/docs/reference/api-guide.md +++ b/docs/reference/api-guide.md @@ -698,6 +698,7 @@ Activates a suspended actor by restoring it onto a physical worker. * `actor`: `ObjectRef` of the actor to resume. * **Response:** `ResumeActorResponse` containing the updated `Actor` object (including the physical worker placement in `status.worker_assignment`). * **Errors:** `RESOURCE_EXHAUSTED` if no eligible worker has room. `FAILED_PRECONDITION` if the actor is `ACTOR_STATE_CRASHED`. When a suspended actor's node is deleted, resume restores its local snapshot on another node from the snapshot's upload, even if atelet never reported the upload complete. Resume does this once another node has room. If atelet finds no upload, resume moves the actor to `ACTOR_STATE_CRASHED` and returns `FAILED_PRECONDITION`. `RevertActor` then returns it to its last completed external snapshot. Revert deletes an upload that was never reported complete, so a snapshot is lost if its restore fails for another reason, or if the actor is reverted before it resumes. + A `FAILED_PRECONDITION` for `ACTOR_STATE_SUSPENDING` or `ACTOR_STATE_REVERTING` also carries an `ErrorInfo` detail with domain `ate.dev` and reason `ACTOR_SUSPENDING` or `ACTOR_REVERTING`, which says the state clears on its own; a caller that fails fast on `FAILED_PRECONDITION`, such as the atenet router, retries those two. #### `SuspendActor` Hibernate a running actor, capturing its current RAM and disk state into a snapshot. From c612328e6fad225c51293137d44dd896a4109a10 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Wed, 28 Oct 2026 09:37:56 -0700 Subject: [PATCH 3/4] atenet/router: park a FailedPrecondition only on a transient reason retryable now parks a FailedPrecondition only when it carries the ACTOR_SUSPENDING or ACTOR_REVERTING reason; every other FailedPrecondition fails at once with the same 503 and description as before. After the commit, note that besides a CRASHED, DELETING or UNSPECIFIED actor, this stops parking on a missing template, a volume without a storage ID, a worker without a node name and a failed DetachActor. Signed-off-by: Yuan Gao --- cmd/atenet/internal/router/ingress/errors.go | 24 ++++ .../internal/router/ingress/errors_test.go | 82 +++++++++++++ .../internal/router/ingress/ingress_test.go | 11 +- cmd/atenet/internal/router/ingress/resumer.go | 15 ++- .../internal/router/ingress/resumer_test.go | 111 ++++++++++++++++-- docs/concepts/request-parking.md | 32 +++-- 6 files changed, 251 insertions(+), 24 deletions(-) diff --git a/cmd/atenet/internal/router/ingress/errors.go b/cmd/atenet/internal/router/ingress/errors.go index 6218d44dfa..0389786516 100644 --- a/cmd/atenet/internal/router/ingress/errors.go +++ b/cmd/atenet/internal/router/ingress/errors.go @@ -20,10 +20,12 @@ import ( "fmt" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/resources" ) @@ -46,6 +48,28 @@ func statusDescription(err error) string { return status.Convert(err).Message() } +// isTransientResumeErr returns true if the error names an actor state a +// concurrent operation moves the actor out of (SUSPENDING, REVERTING), so the +// request may be parked and retried. +func isTransientResumeErr(err error) bool { + type grpcStatus interface{ GRPCStatus() *status.Status } + var gs grpcStatus + if !errors.As(err, &gs) { + return false + } + for _, d := range gs.GRPCStatus().Details() { + info, ok := d.(*errdetails.ErrorInfo) + if !ok || info.GetDomain() != apierror.ErrorInfoDomain { + continue + } + switch apierror.Reason(info.GetReason()) { + case apierror.ReasonActorSuspending, apierror.ReasonActorReverting: + return true + } + } + return false +} + // shedOutcome returns the route outcome of a request that is shed while its // flight retries on retryErr: no_capacity for a ResourceExhausted retryErr, // unavailable otherwise. parkingFullErr and the parking.rejected label both diff --git a/cmd/atenet/internal/router/ingress/errors_test.go b/cmd/atenet/internal/router/ingress/errors_test.go index a95bf1bb86..a4d56cafa6 100644 --- a/cmd/atenet/internal/router/ingress/errors_test.go +++ b/cmd/atenet/internal/router/ingress/errors_test.go @@ -17,16 +17,68 @@ package ingress import ( "context" "errors" + "fmt" "testing" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/resources" ) +// transientPrecondition builds the FailedPrecondition ateapi returns for an +// actor in a state a concurrent operation moves it out of: desc with an +// ErrorInfo detail carrying reason in apierror's domain. +func transientPrecondition(t *testing.T, reason apierror.Reason, desc string) error { + t.Helper() + st, err := status.New(codes.FailedPrecondition, desc).WithDetails(&errdetails.ErrorInfo{Domain: apierror.ErrorInfoDomain, Reason: string(reason)}) + if err != nil { + t.Fatalf("WithDetails: %v", err) + } + return st.Err() +} + +// isTransientResumeErr is true only for apierror's two transient reasons in +// its own domain; any other detail, a status with none, or a non-status error +// fails fast. +func TestIsTransientResumeErr(t *testing.T) { + t.Parallel() + + withDetail := func(domain, reason string) error { + t.Helper() + st, err := status.New(codes.FailedPrecondition, "actor is "+reason).WithDetails(&errdetails.ErrorInfo{Domain: domain, Reason: reason}) + if err != nil { + t.Fatalf("WithDetails: %v", err) + } + return st.Err() + } + tests := []struct { + name string + err error + want bool + }{ + {"plain FailedPrecondition", status.Error(codes.FailedPrecondition, "actor is ACTOR_STATE_CRASHED"), false}, + {"ACTOR_SUSPENDING in the ate.dev domain", transientPrecondition(t, apierror.ReasonActorSuspending, "actor is ACTOR_STATE_SUSPENDING"), true}, + {"ACTOR_REVERTING in the ate.dev domain", transientPrecondition(t, apierror.ReasonActorReverting, "actor is ACTOR_STATE_REVERTING"), true}, + {"matching reason in another domain", withDetail("example.com", string(apierror.ReasonActorSuspending)), false}, + {"unknown reason in the ate.dev domain", withDetail(apierror.ErrorInfoDomain, "ACTOR_CRASHED"), false}, + {"wrapped transient status", fmt.Errorf("while resuming: %w", transientPrecondition(t, apierror.ReasonActorSuspending, "actor is ACTOR_STATE_SUSPENDING")), true}, + {"non-status error", errors.New("resume failed"), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := isTransientResumeErr(tt.err); got != tt.want { + t.Errorf("isTransientResumeErr(%v) = %t, want %t", tt.err, got, tt.want) + } + }) + } +} + func TestActorNotFoundErr(t *testing.T) { t.Parallel() @@ -84,6 +136,18 @@ func TestMapResumeError(t *testing.T) { actorRef := resources.ActorRef{Atespace: "team-a", Name: "ctr6"} + // otherDomainPrecondition carries an ErrorInfo whose reason matches but + // whose domain is not apierror's; the router maps it like any other + // FailedPrecondition. + otherDomainPrecondition := func(t *testing.T) error { + t.Helper() + st, err := status.New(codes.FailedPrecondition, "actor is ACTOR_STATE_SUSPENDING").WithDetails(&errdetails.ErrorInfo{Domain: "example.com", Reason: string(apierror.ReasonActorSuspending)}) + if err != nil { + t.Fatalf("WithDetails: %v", err) + } + return st.Err() + } + tests := []struct { name string err error @@ -102,6 +166,24 @@ func TestMapResumeError(t *testing.T) { wantCode: envoy_type.StatusCode_ServiceUnavailable, wantBody: `actor team-a/ctr6 unavailable: no worker has room for the actor`, }, + { + name: "FailedPrecondition with the ACTOR_SUSPENDING reason after the budget maps to 503 and preserves desc", + err: &budgetExhaustedError{lastErr: transientPrecondition(t, apierror.ReasonActorSuspending, "actor team-a/ctr6 is ACTOR_STATE_SUSPENDING")}, + wantCode: envoy_type.StatusCode_ServiceUnavailable, + wantBody: `actor team-a/ctr6 unavailable: actor team-a/ctr6 is ACTOR_STATE_SUSPENDING`, + }, + { + name: "FailedPrecondition with the ACTOR_REVERTING reason maps to 503", + err: transientPrecondition(t, apierror.ReasonActorReverting, "actor team-a/ctr6 is ACTOR_STATE_REVERTING"), + wantCode: envoy_type.StatusCode_ServiceUnavailable, + wantBody: `actor team-a/ctr6 unavailable: actor team-a/ctr6 is ACTOR_STATE_REVERTING`, + }, + { + name: "FailedPrecondition with a reason from another domain maps to 503", + err: otherDomainPrecondition(t), + wantCode: envoy_type.StatusCode_ServiceUnavailable, + wantBody: `actor team-a/ctr6 unavailable: actor is ACTOR_STATE_SUSPENDING`, + }, { name: "Unavailable maps to 503", err: status.Error(codes.Unavailable, "control-plane down"), diff --git a/cmd/atenet/internal/router/ingress/ingress_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go index e27475b34a..971a4f7bc8 100644 --- a/cmd/atenet/internal/router/ingress/ingress_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -35,6 +35,7 @@ import ( "go.ate.dev/substrate/api/ateapi/v1" "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/atenet" atunnelingress "go.ate.dev/substrate/internal/atunnel/ingress" @@ -305,6 +306,14 @@ func TestHandleRequestHeaders(t *testing.T) { expectedErrStr: `actor team-a/123e4567-e89b-12d3-a456-426614174000 unavailable: no worker has room for the actor`, expectedStatus: envoy_type.StatusCode_ServiceUnavailable, }, + { + name: "FailedPrecondition with a transient reason and parking disabled maps to 503 at once", + authority: testUUID + ".team-a.actors.resources.substrate.ate.dev", + resumeErr: transientPrecondition(t, apierror.ReasonActorSuspending, "actor is ACTOR_STATE_SUSPENDING"), + expectErr: true, + expectedErrStr: `actor team-a/123e4567-e89b-12d3-a456-426614174000 unavailable: actor is ACTOR_STATE_SUSPENDING`, + expectedStatus: envoy_type.StatusCode_ServiceUnavailable, + }, { name: "NotFound maps to 404", authority: testUUID + ".team-a.actors.resources.substrate.ate.dev", @@ -809,7 +818,7 @@ func TestHandleRequestHeaders_FullLotShedsParkedRequest(t *testing.T) { }, { name: "FailedPrecondition is dropped", - resumeErr: status.Error(codes.FailedPrecondition, "actor is suspending"), + resumeErr: transientPrecondition(t, apierror.ReasonActorSuspending, "actor is suspending"), wantCode: codes.Unknown, wantOutcome: ateattr.RouterOutcomeUnavailable, }, diff --git a/cmd/atenet/internal/router/ingress/resumer.go b/cmd/atenet/internal/router/ingress/resumer.go index 0ccbff2cea..c792f63b55 100644 --- a/cmd/atenet/internal/router/ingress/resumer.go +++ b/cmd/atenet/internal/router/ingress/resumer.go @@ -103,9 +103,8 @@ type ActorResumer struct { // never on the fast path (#1081). A nil lot admits everyone. lot *parkingLot - // parkEnabled makes transient worker-pool saturation (FailedPrecondition) - // retryable, so a request is parked and retried until budget rather than - // failing immediately. + // parkEnabled lets retryable park a request on a transient condition and + // retry it until budget rather than failing immediately. parkEnabled bool // budget bounds the total time a single resume operation retries before the // underlying error is returned. @@ -164,16 +163,20 @@ func NewActorResumer(apiClient ateapi.ControlClient, opts ...resumerOption) *Act // ateapi rolling restart) are retried only when parking is enabled, turning a // momentary condition into a bounded wait instead of an immediate failure — a // parked request should ride out a blip, not fail on it with budget remaining. -// FailedPrecondition stays retryable too: it no longer carries saturation, but -// it does cover states a concurrent operation can move the actor out of. +// FailedPrecondition is retried only when parking is enabled and the actor is +// in a state a concurrent operation moves it out of (SUSPENDING, REVERTING); +// see [isTransientResumeErr]. Every other FailedPrecondition, such as a +// CRASHED actor, fails fast, as does a transient one while parking is disabled. // All other codes (NotFound, DeadlineExceeded, PermissionDenied, ...) are // returned to the caller so the HTTP boundary can map them with full fidelity. func (r *ActorResumer) retryable(err error) bool { switch status.Code(err) { case codes.Aborted: return true - case codes.ResourceExhausted, codes.FailedPrecondition, codes.Unavailable: + case codes.ResourceExhausted, codes.Unavailable: return r.parkEnabled + case codes.FailedPrecondition: + return r.parkEnabled && isTransientResumeErr(err) default: return false } diff --git a/cmd/atenet/internal/router/ingress/resumer_test.go b/cmd/atenet/internal/router/ingress/resumer_test.go index c37a0f9254..eeb3df28bd 100644 --- a/cmd/atenet/internal/router/ingress/resumer_test.go +++ b/cmd/atenet/internal/router/ingress/resumer_test.go @@ -27,6 +27,7 @@ import ( envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" "go.ate.dev/substrate/api/ateapi/v1" "go.ate.dev/substrate/cmd/atenet/internal/router/extproc" + "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/resources" "go.opentelemetry.io/otel/trace" @@ -328,7 +329,7 @@ func TestActorResumer_Parking(t *testing.T) { mu.Unlock() if n < 3 { // Worker pool momentarily saturated. - return nil, status.Error(codes.FailedPrecondition, "no worker has room for the actor") + return nil, status.Error(codes.ResourceExhausted, "no worker has room for the actor") } return &ateapi.ResumeActorResponse{ Actor: &ateapi.Actor{Metadata: &ateapi.ResourceMetadata{Name: testActorName}, Status: &ateapi.ActorStatus{State: ateapi.ActorState_ACTOR_STATE_RUNNING, WorkerAssignment: &ateapi.WorkerAssignment{WorkerPodIps: []string{expectedIP}}}}, @@ -361,7 +362,7 @@ func TestActorResumer_Parking(t *testing.T) { mu.Lock() calls++ mu.Unlock() - return nil, status.Error(codes.FailedPrecondition, "no worker has room for the actor") + return nil, status.Error(codes.ResourceExhausted, "no worker has room for the actor") }, } @@ -371,8 +372,8 @@ func TestActorResumer_Parking(t *testing.T) { _, _, err := resumer.ResumeActor(context.Background(), testActorRef) // The client must see the meaningful capacity error, not a generic // timeout: status.Code must unwrap through the budget-exhaustion marker. - if got := status.Code(err); got != codes.FailedPrecondition { - t.Errorf("expected FailedPrecondition after park budget elapsed, got %v (err=%v)", got, err) + if got := status.Code(err); got != codes.ResourceExhausted { + t.Errorf("expected ResourceExhausted after park budget elapsed, got %v (err=%v)", got, err) } var budget *budgetExhaustedError if !errors.As(err, &budget) { @@ -386,6 +387,64 @@ func TestActorResumer_Parking(t *testing.T) { }) }) + t.Run("PlainPreconditionFailsFast", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var calls atomic.Int32 + mock := &resumerMockClient{ + resumeFn: func(ctx context.Context, in *ateapi.ResumeActorRequest, opts ...grpc.CallOption) (*ateapi.ResumeActorResponse, error) { + calls.Add(1) + // No ErrorInfo reason: a CRASHED actor, which only RevertActor moves on. + return nil, status.Error(codes.FailedPrecondition, "AssignWorker prerequisite not met for Actor: team-a/ctr6 (got: ACTOR_STATE_CRASHED, want ACTOR_STATE_SUSPENDED)") + }, + } + resumer := NewActorResumer(mock, withParking(ParkedRequestConfig{Max: 1, Budget: 5 * time.Second})) + start := time.Now() + _, _, err := resumer.ResumeActor(context.Background(), testActorRef) + if got := status.Code(err); got != codes.FailedPrecondition { + t.Errorf("code = %v, want FailedPrecondition (err=%v)", got, err) + } + var budget *budgetExhaustedError + if errors.As(err, &budget) { + t.Errorf("plain precondition was treated as budget exhaustion: %v", err) + } + if got := calls.Load(); got != 1 { + t.Errorf("resume attempts = %d, want 1", got) + } + if waited := time.Since(start); waited != 0 { + t.Errorf("waited %v, want no retry delay", waited) + } + }) + }) + + t.Run("ParksOnTransientPrecondition", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var calls atomic.Int32 + mock := &resumerMockClient{ + resumeFn: func(ctx context.Context, in *ateapi.ResumeActorRequest, opts ...grpc.CallOption) (*ateapi.ResumeActorResponse, error) { + if calls.Add(1) < 3 { + // A suspend still in flight; ateapi marks it transient. + return nil, transientPrecondition(t, apierror.ReasonActorSuspending, "actor is ACTOR_STATE_SUSPENDING") + } + return &ateapi.ResumeActorResponse{ + Actor: &ateapi.Actor{Metadata: &ateapi.ResourceMetadata{Name: testActorName}, Status: &ateapi.ActorStatus{State: ateapi.ActorState_ACTOR_STATE_RUNNING, WorkerAssignment: &ateapi.WorkerAssignment{WorkerPodIps: []string{expectedIP}}}}, + Resumed: true, + }, nil + }, + } + resumer := NewActorResumer(mock, withParking(ParkedRequestConfig{Max: 1, Budget: 5 * time.Second})) + actor, _, err := resumer.ResumeActor(context.Background(), testActorRef) + if err != nil { + t.Fatalf("a transient precondition must be parked and retried, got: %v", err) + } + if got := calls.Load(); got != 3 { + t.Errorf("resume attempts = %d, want 3", got) + } + if !slices.Equal(actor.GetStatus().GetWorkerAssignment().GetWorkerPodIps(), []string{expectedIP}) { + t.Errorf("expected IP %q, got %q", expectedIP, actor.GetStatus().GetWorkerAssignment().GetWorkerPodIps()) + } + }) + }) + t.Run("ParksThroughUnavailableBlip", func(t *testing.T) { synctest.Test(t, func(t *testing.T) { var mu sync.Mutex @@ -589,15 +648,15 @@ func TestActorResumer_Parking(t *testing.T) { mu.Lock() calls++ mu.Unlock() - return nil, status.Error(codes.FailedPrecondition, "no worker has room for the actor") + return nil, status.Error(codes.ResourceExhausted, "no worker has room for the actor") }, } // Default constructor => parking disabled => fail-fast. resumer := NewActorResumer(mock) _, _, err := resumer.ResumeActor(context.Background(), testActorRef) - if got := status.Code(err); got != codes.FailedPrecondition { - t.Errorf("expected FailedPrecondition, got %v (err=%v)", got, err) + if got := status.Code(err); got != codes.ResourceExhausted { + t.Errorf("expected ResourceExhausted, got %v (err=%v)", got, err) } mu.Lock() defer mu.Unlock() @@ -757,6 +816,44 @@ func TestActorResumer_LotAdmission(t *testing.T) { }) }) + t.Run("PlainPreconditionNeverEntersLot", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + // A full lot sheds any caller that tries to park, so a plain + // FailedPrecondition must come back as itself, not as the shed. + cfg := ParkedRequestConfig{Max: 1, Budget: 5 * time.Second} + lot := newParkingLot(cfg, nil) + release, ok := lot.enter(context.Background(), ateattr.RouterOutcomeUnavailable) + if !ok { + t.Fatal("priming enter should be admitted") + } + defer release(parkOutcomeServed) + + var calls atomic.Int32 + mock := &resumerMockClient{ + resumeFn: func( + ctx context.Context, + in *ateapi.ResumeActorRequest, + opts ...grpc.CallOption, + ) (*ateapi.ResumeActorResponse, error) { + calls.Add(1) + return nil, status.Error(codes.FailedPrecondition, "AssignWorker prerequisite not met for Actor: team-a/ctr6 (got: ACTOR_STATE_CRASHED, want ACTOR_STATE_SUSPENDED)") + }, + } + resumer := NewActorResumer(mock, withParking(cfg), withParkingLot(lot)) + _, _, err := resumer.ResumeActor(context.Background(), testActorRef) + var reqErr *extproc.ReqError + if errors.As(err, &reqErr) { + t.Fatalf("a plain precondition was shed by the full lot, so it tried to park: %v", err) + } + if got := status.Code(err); got != codes.FailedPrecondition { + t.Errorf("code = %v, want FailedPrecondition (err=%v)", got, err) + } + if got := calls.Load(); got != 1 { + t.Errorf("resume attempts = %d, want 1", got) + } + }) + }) + t.Run("ParkTransitionAcquiresSlot", func(t *testing.T) { synctest.Test(t, func(t *testing.T) { cfg := ParkedRequestConfig{Max: 2, Budget: 5 * time.Second} diff --git a/docs/concepts/request-parking.md b/docs/concepts/request-parking.md index 5ff2528a94..cdfe9db6db 100644 --- a/docs/concepts/request-parking.md +++ b/docs/concepts/request-parking.md @@ -38,12 +38,16 @@ user-visible error. ## Behavior -With parking enabled (the default), the router treats `ResourceExhausted`, -`FailedPrecondition` and `Unavailable` from `ResumeActor` as **retryable** -conditions (alongside the existing `Aborted` concurrent-resume conflict) — a -parked request rides out transient pool saturation and control-plane blips -(e.g. an ateapi rolling restart) alike. The request is *parked*: the resumer -keeps retrying with exponential backoff until either +With parking enabled (the default), the router treats `ResourceExhausted` and +`Unavailable` from `ResumeActor` as **retryable** conditions (alongside the +existing `Aborted` concurrent-resume conflict). `FailedPrecondition` covers +several cases and only two of them are retryable: a suspend or a revert still +in flight, which ateapi marks with the `ErrorInfo` reason `ACTOR_SUSPENDING` or +`ACTOR_REVERTING`; every other `FailedPrecondition` fails fast (see below). A +parked request rides out transient pool saturation, control-plane blips (e.g. +an ateapi rolling restart) and a suspend or revert still in flight alike. The +request is *parked*: the resumer keeps retrying with exponential backoff until +either - the resume succeeds (the actor is `RUNNING` and has a worker IP) — the request is then routed normally; or @@ -51,6 +55,12 @@ keeps retrying with exponential backoff until either underlying capacity error is returned, surfacing as `503 "actor unavailable: no worker has room for the actor"`. +Every other `FailedPrecondition` is not parked. Without one of those two +reasons (domain `ate.dev`) the actor's state does not clear on its own, for +example `ACTOR_STATE_CRASHED`, which only `RevertActor` moves on. The request +fails at once with `503` and the description from ateapi, and no parking slot +is taken. + **The budget bounds retries, not a committed resume.** When the budget elapses the router stops starting new resume attempts, but an attempt already in flight is **never canceled**: by then the control plane has committed work to @@ -112,16 +122,18 @@ expected under sustained saturation.) ### What is *not* parked Only transient conditions — capacity (`ResourceExhausted`), transient actor -state (`FailedPrecondition`), concurrency (`Aborted`), and control-plane -unavailability (`Unavailable`) — are parked. Errors that will not resolve by -waiting are returned immediately (fail fast): +state (`FailedPrecondition` with reason `ACTOR_SUSPENDING` or +`ACTOR_REVERTING`), concurrency (`Aborted`), and control-plane unavailability +(`Unavailable`) — are parked. Errors that will not resolve by waiting are +returned immediately (fail fast): | Resume result | Behavior | | -------------------------------------- | --------------------------------- | | `OK` | Route to worker | | `Aborted` (concurrent resume) | Retry (always) | | `ResourceExhausted` (no room) | **Park & retry** (when enabled) | -| `FailedPrecondition` (transient state) | **Park & retry** (when enabled) | +| `FailedPrecondition` with reason `ACTOR_SUSPENDING` or `ACTOR_REVERTING` | **Park & retry** (when enabled) | +| `FailedPrecondition` otherwise (e.g. `ACTOR_STATE_CRASHED`) | Fail fast → `503` | | `Unavailable` (control-plane blip) | **Park & retry** (when enabled) | | `NotFound` | Fail fast → `404` | | `DeadlineExceeded` | Fail fast → `504` | From 86f55818aee3c69e06fd1ed126b8667096aa70c4 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Wed, 28 Oct 2026 09:37:56 -0700 Subject: [PATCH 4/4] e2e: test that a routed request to a CRASHED actor fails fast The new demo test crashes an actor by deleting its worker pod, routes one request and expects 503 with ACTOR_STATE_CRASHED in the body. On Envoy, the answer must arrive in under 2 s to prove nothing parked. AgentGateway resumes through ate-api-server itself and still parks, so the dataplane's CrashedActorFailsFast flag skips the time bound there. Signed-off-by: Yuan Gao --- internal/e2e/atenet_dataplane.go | 10 ++ .../e2e/suites/demo/crashed_request_test.go | 133 ++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 internal/e2e/suites/demo/crashed_request_test.go diff --git a/internal/e2e/atenet_dataplane.go b/internal/e2e/atenet_dataplane.go index 5267897a1f..567418efe7 100644 --- a/internal/e2e/atenet_dataplane.go +++ b/internal/e2e/atenet_dataplane.go @@ -34,6 +34,9 @@ type AtenetDataplane interface { NewParkingObserver(context.Context) (ParkingObserver, error) IsRetryableParkingBudgetExhaustion(status int, body string) bool ParkingBudgetStatus() int + // CrashedActorFailsFast reports whether a routed request to an actor in + // ACTOR_STATE_CRASHED comes back without parking. + CrashedActorFailsFast() bool IsEgressPolicyDenied(status int, body string) bool PlatformMetricPrefixes([]string) []string RouteDurationSeen(context.Context, string) (bool, error) @@ -76,6 +79,9 @@ func (envoyAtenetDataplane) IsRetryableParkingBudgetExhaustion(status int, body func (envoyAtenetDataplane) ParkingBudgetStatus() int { return http.StatusServiceUnavailable } +// Envoy's router answers 503 at once. +func (envoyAtenetDataplane) CrashedActorFailsFast() bool { return true } + func (envoyAtenetDataplane) IsEgressPolicyDenied(status int, body string) bool { return isEgressPolicyDenied(status, body) } @@ -145,6 +151,10 @@ func (agentGatewayAtenetDataplane) IsRetryableParkingBudgetExhaustion(status int func (agentGatewayAtenetDataplane) ParkingBudgetStatus() int { return http.StatusServiceUnavailable } +// AgentGateway resumes through ate-api-server itself, parks for its budget, +// and answers 503. +func (agentGatewayAtenetDataplane) CrashedActorFailsFast() bool { return false } + func (agentGatewayAtenetDataplane) IsEgressPolicyDenied(status int, body string) bool { return isEgressPolicyDenied(status, body) } diff --git a/internal/e2e/suites/demo/crashed_request_test.go b/internal/e2e/suites/demo/crashed_request_test.go new file mode 100644 index 0000000000..4b5fca5ce2 --- /dev/null +++ b/internal/e2e/suites/demo/crashed_request_test.go @@ -0,0 +1,133 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package demo + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "go.ate.dev/substrate/api/ateapi/v1" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// A routed request to a CRASHED actor must fail fast: the router does not park +// it or retry ResumeActor, since only RevertActor moves the actor on. Not +// parallel: the timing bound is the proof that nothing parked. +func TestRoutedRequestToCrashedActorFailsFast(t *testing.T) { + nsObj := e2e.CreateNamespace(t) + ctx := context.Background() + clients := e2e.GetClients() + + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) + if err != nil { + t.Fatalf("failed to initialize ActorTemplate: %v", err) + } + + actorName := "crashed-request-actor-" + nsObj.Name + actorRef := &ateapi.ObjectRef{Atespace: demoAtespace, Name: actorName} + routedRef := resources.ActorRef{Atespace: demoAtespace, Name: actorName} + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + _, _ = clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapi.DeleteActorRequest{Actor: actorRef}) + }) + + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapi.CreateActorRequest{Actor: &ateapi.Actor{ + Metadata: &ateapi.ResourceMetadata{Atespace: demoAtespace, Name: actorName}, + ActorTemplate: e2e.TemplateRef(at), + }}); err != nil { + t.Fatalf("failed to create Actor: %v", err) + } + waitForActorState(ctx, t, clients, actorName, ateapi.ActorState_ACTOR_STATE_SUSPENDED) + + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapi.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("failed to resume Actor: %v", err) + } + waitForActorState(ctx, t, clients, actorName, ateapi.ActorState_ACTOR_STATE_RUNNING) + + resp, err := callActor(t, routedRef) + if err != nil { + t.Fatalf("failed to call actor before crash: %v", err) + } + validateCounterResponse(t, resp, "before crash", 1, 1) + + actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapi.GetActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("failed to get Actor: %v", err) + } + podName := actor.GetStatus().GetWorkerAssignment().GetWorkerPod() + podNamespace := actor.GetStatus().GetWorkerAssignment().GetWorkerNamespace() + if podName == "" || podNamespace == "" { + t.Fatalf("running actor has no pod recorded: %v", actor.GetStatus().GetWorkerAssignment()) + } + + // Deleting the pod is how a real crash happens, as in TestRevertCrashedActor. + if err := clients.K8s.CoreV1().Pods(podNamespace).Delete(ctx, podName, metav1.DeleteOptions{}); err != nil { + t.Fatalf("failed to delete worker pod %s/%s: %v", podNamespace, podName, err) + } + waitForActorState(ctx, t, clients, actorName, ateapi.ActorState_ACTOR_STATE_CRASHED) + + router, err := e2e.NewRouterClient(t, ctx) + if err != nil { + t.Fatalf("connecting to atenet-router: %v", err) + } + t.Cleanup(router.Close) + + reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + start := time.Now() + httpResp, err := router.Do(reqCtx, http.MethodPost, routedRef, "/", nil) + elapsed := time.Since(start) + if err != nil { + t.Fatalf("routed request to CRASHED actor failed at transport level after %v: %v", elapsed, err) + } + defer httpResp.Body.Close() + b, _ := io.ReadAll(httpResp.Body) + body := string(b) + t.Logf("routed request to CRASHED actor: status=%d elapsed=%v body=%q", httpResp.StatusCode, elapsed, body) + + if httpResp.StatusCode != http.StatusServiceUnavailable { + t.Errorf("status = %d, want %d", httpResp.StatusCode, http.StatusServiceUnavailable) + } + if !strings.Contains(body, "ACTOR_STATE_CRASHED") { + t.Errorf("body = %q, want it to name ACTOR_STATE_CRASHED", body) + } + failsFast := e2e.CurrentAtenetDataplane().CrashedActorFailsFast() + // A parked request waits out the router's parking budget. Well under it + // proves the router answered on the first ResumeActor attempt. + if failsFast && elapsed >= 2*time.Second { + t.Errorf("request took %v, want under 2s: the router parked a request no retry can serve", elapsed) + } + + // The route works again once an operator reverts the actor. + if _, err := clients.SubstrateAPI.RevertActor(ctx, &ateapi.RevertActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("RevertActor from CRASHED: %v", err) + } + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapi.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("failed to resume reverted Actor: %v", err) + } + waitForActorState(ctx, t, clients, actorName, ateapi.ActorState_ACTOR_STATE_RUNNING) + resp, err = callActor(t, routedRef) + if err != nil { + t.Fatalf("failed to call actor after revert: %v", err) + } + validateCounterResponse(t, resp, "after revert", 1, 1) +}