Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apps/daemon/internal/dispatch/runtime_preparation.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,9 @@ func (r *Router) runRuntimePreparationTransfer(ctx context.Context, u *runtimePr
r.mu.Unlock()
result := rejectedRuntimePreparation("invalid_request")
if admitted {
started := time.Now()
result = runtimePreparationResult(apply(ctx, u.request, data), u.request.SizeBytes)
result.DurationMS = time.Since(started).Milliseconds()
}
// Release the potentially large body before waiting on transport delivery.
data = nil
Expand Down
22 changes: 22 additions & 0 deletions apps/daemon/internal/dispatch/runtime_preparation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -441,3 +441,25 @@ func TestRuntimePreparationHonorsBeginBudgetDuringStaging(t *testing.T) {
shutdownCapabilitiesRouter(t, r)
})
}

func TestRuntimePreparationReportsApplyDuration(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
r, sender, environment, session := capabilitiesTestRouter(t)
_, cancel := context.WithCancel(t.Context())
id := uuid.NewString()
request := proto.RuntimePreparePayload{BudgetMS: 300000, Step: "begin", Action: "finalize", EnvironmentID: environment, SessionID: session, Sources: &proto.CapabilitySources{}}
owner := &runtimePreparationTransfer{envelope: capabilityEnvelope(t, id, request), request: request, ready: make(chan struct{}), cancel: cancel, finished: true, apply: true}
close(owner.ready)
r.runtimePreparation = owner
r.shutdownWG.Add(1)
go r.runRuntimePreparationTransfer(t.Context(), owner, func(context.Context, proto.RuntimePreparePayload, []byte) error {
time.Sleep(1500 * time.Millisecond)
return nil
})
synctest.Wait()
got := capabilitiesReceipt(t, sender, id, "completed")
if got.DurationMS != 1500 || !proto.ValidRuntimePrepareResult(got, "completed", 0, 0) {
t.Fatalf("apply duration not reported: %+v", got)
}
})
}
2 changes: 1 addition & 1 deletion apps/web/e2e/fixture-console.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ function adminRead(response, path, url) {
if (m[2] === "diagnostics") {
const failedTurn = (state.turns.get(session.id) ?? []).findLast((turn) => turn.status === "failed");
return send(response, 200, { object: "core.session_diagnostics", session_id: session.id, status: session.status,
failure: session.status === "failed" ? { source: failedTurn ? "turn" : "environment_input", ...(failedTurn ? { turn_id: failedTurn.id } : {}), code: "internal_error", params: {}, failed_at: null } : null });
failure: session.status === "failed" ? { source: failedTurn ? "turn" : "environment_input", ...(failedTurn ? { turn_id: failedTurn.id } : {}), code: "internal_error", params: {}, failed_at: null } : null, initialization_steps: [] });
}
const diagnostics = m[2].match(/^turns\/([^/]+)\/diagnostics$/);
if (diagnostics) {
Expand Down
3 changes: 2 additions & 1 deletion apps/web/src/features/sessions/SessionPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { sessionTitle } from "../../lib/session-title";
import { forgetDeleted } from "../resources/detail-queries";
import { ExecutorCredentialsSection } from "./ExecutorCredentialsSection";
import { SessionDeleteDialog, type SessionDeleteTarget } from "./SessionDeleteDialog";
import { DiagnosticScope, SessionFailureDetails } from "./session-diagnostics";
import { DiagnosticScope, InitializationSteps, SessionFailureDetails } from "./session-diagnostics";
import { sessionKey } from "./session-queries";
import { isSessionActive, turnAnchorId } from "./session-history";
import { environmentKind, isDeletable } from "./session-log";
Expand Down Expand Up @@ -150,6 +150,7 @@ export function SessionPage() {
{session.vault_ids.length ? session.vault_ids.map((id) => <CopyableId key={id} id={id} compact />) : MISSING}
</dd>
</div>
<InitializationSteps projectId={projectId} session={session} />
{waiting.length ? <div><dt>{t("detail.waitingFor")}</dt><dd className="session-fact-stack">{waiting.map((entry, index) => <span key={index}>{entry}</span>)}{session.required_actions.some((action) => action.type === "function_call") ? <span className="detail-note">{t("detail.applicationAction")}</span> : null}</dd></div> : null}
{metadata.length ? (
<div>
Expand Down
17 changes: 14 additions & 3 deletions apps/web/src/features/sessions/session-diagnostics.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@ import { afterEach, describe, expect, it } from "vitest";

import i18n from "../../i18n";
import { diagnosticFailure } from "../../lib/diagnostic-failure";
import { DiagnosticScope, SessionFailure, SessionFailureDetails, TurnFailure, sessionDiagnosticsQuery, turnDiagnosticsQuery } from "./session-diagnostics";
import { DiagnosticScope, InitializationSteps, SessionFailure, SessionFailureDetails, TurnFailure, sessionDiagnosticsQuery, turnDiagnosticsQuery } from "./session-diagnostics";
import { ItemReceiptTiming } from "./trace/ItemReceiptTiming";

const scope = { projectId: "project", sessionId: "session" };
const session = { id: "session", status: "failed", last_active_at: 20 } as AgentSession;
const turn = { id: "turn", session_id: "session", status: "failed", completed_at: 20, error: { code: "internal_error", message: "PRIVATE_NATIVE_SENTINEL" } } as AgentTurn;
const failure: DiagnosticFailure = { code: "authentication_error", params: {}, failed_at: null };
const sessionData: SessionDiagnostics = { object: "core.session_diagnostics", session_id: "session", status: "failed", failure: { ...failure, source: "turn", turn_id: "turn" } };
const sessionData: SessionDiagnostics = { object: "core.session_diagnostics", session_id: "session", status: "failed", failure: { ...failure, source: "turn", turn_id: "turn" }, initialization_steps: [{ step: "npm", index: null, duration_ms: 4100 }, { step: "setup", index: 1, duration_ms: 5200 }] };
const turnData: TurnDiagnostics = { object: "core.turn_diagnostics", session_id: "session", turn_id: "turn", status: "failed", failure, items: [], items_truncated: false };

afterEach(() => { void i18n.changeLanguage("en"); });
Expand All @@ -38,6 +38,17 @@ describe("Core diagnostics in the console", () => {
expect(render("turn", turnData)).not.toContain("PRIVATE_NATIVE_SENTINEL");
});

it("lists the initialization operations that ran with their durations and the failure category", () => {
const client = new QueryClient();
client.setQueryData(sessionDiagnosticsQuery(scope.projectId, session).queryKey, sessionData);
const html = renderToStaticMarkup(<QueryClientProvider client={client}><InitializationSteps projectId={scope.projectId} session={session} /></QueryClientProvider>);
client.clear();
expect(html).toContain("npm dependencies: 4.1 s");
expect(html).toContain("setup 2: 5.2 s");
const t = i18n.getFixedT("en", "diagnostics");
expect(diagnosticFailure({ code: "environment_provisioning_failed", params: { step: null, index: null, exit_code: null, category: "timeout" }, failed_at: null }, t)).toContain("The preparation timed out.");
});

it("gives the Session page the reason, then its next step, in one block", () => {
expect(render("session", sessionData)).toContain('<p class="session-failure-reason">The model provider rejected authentication.</p><p>Check the API key');
});
Expand All @@ -58,7 +69,7 @@ describe("Core diagnostics in the console", () => {

it("renders only typed provisioning metadata and does not guess a timeout", () => {
const t = i18n.getFixedT("en", "diagnostics");
const text = diagnosticFailure({ code: "environment_provisioning_failed", params: { step: "setup", index: 0, exit_code: 7 }, failed_at: null }, t);
const text = diagnosticFailure({ code: "environment_provisioning_failed", params: { step: "setup", index: 0, exit_code: 7, category: null }, failed_at: null }, t);
expect(text).toContain("Setup command 1");
expect(text).toContain("Exit code 7");
expect(diagnosticFailure({ code: "harness_error", params: {}, failed_at: null }, t)).not.toContain("timed out");
Expand Down
14 changes: 13 additions & 1 deletion apps/web/src/features/sessions/session-diagnostics.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { createContext, useContext } from "react";
import { useTranslation } from "react-i18next";

import { HelpTip } from "../../components/console-ui";
import { formatDuration } from "../../lib/format";
import { diagnosticFailure, diagnosticNext, diagnosticReason } from "../../lib/diagnostic-failure";
import { admin } from "../../lib/projects";
import { sessionKey } from "./session-queries";
Expand All @@ -13,7 +14,7 @@ export const DiagnosticScope = createContext<Scope | null>(null);

export function sessionDiagnosticsQuery(projectId: string, session: AgentSession) {
return queryOptions({
queryKey: [...sessionKey(projectId, session.id), "diagnostics", "session", session.status, session.last_active_at],
queryKey: [...sessionKey(projectId, session.id), "diagnostics", "session", session.status],
queryFn: ({ signal }) => admin.retrieveSessionDiagnostics(projectId, session.id, { signal }),
enabled: session.status === "failed",
retry: false,
Expand All @@ -39,6 +40,17 @@ export function DiagnosticsUnavailable({ pending, retry }: { pending: boolean; r
</span>;
}

/** The facts row of the operations the Environment's initialization ran, in order. */
export function InitializationSteps({ projectId, session }: { projectId: string; session: AgentSession }) {
const { t } = useTranslation("diagnostics");
const query = useQuery({ ...sessionDiagnosticsQuery(projectId, session), enabled: true });
const steps = query.isError ? [] : query.data?.initialization_steps ?? [];
if (!steps.length) return null;
return <div><dt>{t("initializationTitle")}</dt><dd className="session-fact-stack">
{steps.map((step, index) => <span key={index}>{t(`step.${step.step}`)}{step.index === null ? "" : ` ${step.index + 1}`}: {formatDuration(step.duration_ms / 1000)}</span>)}
</dd></div>;
}

function useSessionFailure(projectId: string, session: AgentSession) {
const query = useQuery(sessionDiagnosticsQuery(projectId, session));
return { query, failure: !query.isError && query.data?.status === session.status ? query.data.failure : null };
Expand Down
15 changes: 14 additions & 1 deletion apps/web/src/i18n/locales/en/diagnostics.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import type { ProvisioningCategory } from "@oac/agents-client";

export const diagnostics = {
providerUnknown: "Not recorded yet",
providerStale: "Observation unconfirmed",
Expand Down Expand Up @@ -61,7 +63,18 @@ export const diagnostics = {
exitCode: "Exit code {{code}}.",
httpStatus: "HTTP {{status}}.",
unreachableHost: "Then check that the URL's host resolves and is reachable from the Session's sandbox, or from the host of a self-hosted Session.",
step: { setup: "setup", python: "Python dependencies", npm: "npm dependencies", file: "file preparation", skill: "Skill preparation", harness: "Harness preparation" },
step: { setup: "setup", python: "Python dependencies", npm: "npm dependencies", file: "file preparation", skill: "Skill preparation", harness: "Harness preparation", plugin: "Plugin preparation", configure: "environment configuration", finalize: "finalization" },
category: {
timeout: "The preparation timed out.",
runtime_disconnected: "The Runtime disconnected.",
unconfirmed: "The Runtime did not confirm the result.",
rejected: "The Runtime rejected the operation.",
operation_failed: "The Runtime reported a failed operation.",
core_error: "Core could not read, validate or record the initialization state.",
core_restart: "Core lost ownership of the initialization.",
compute_released: "The sandbox was released or never created.",
} satisfies Record<ProvisioningCategory, string>,
initializationTitle: "Initialization",
unavailable: "Failure details unavailable",
unavailableHelp: "The diagnostics read failed or no longer matches this status. Refresh to check again.",
checking: "Checking failure details",
Expand Down
15 changes: 14 additions & 1 deletion apps/web/src/i18n/locales/zh-CN/diagnostics.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import type { ProvisioningCategory } from "@oac/agents-client";

export const diagnostics = {
providerUnknown: "还没有记录",
providerStale: "记录尚未确认",
Expand Down Expand Up @@ -61,7 +63,18 @@ export const diagnostics = {
exitCode: "退出码 {{code}}。",
httpStatus: "HTTP {{status}}。",
unreachableHost: "再确认 Session 的沙箱(自托管 Session 则为其主机)能解析并访问该 URL 的主机。",
step: { setup: "设置", python: "Python 依赖", npm: "npm 依赖", file: "文件准备", skill: "Skill 准备", harness: "Harness 准备" },
step: { setup: "设置", python: "Python 依赖", npm: "npm 依赖", file: "文件准备", skill: "Skill 准备", harness: "Harness 准备", plugin: "Plugin 准备", configure: "环境配置", finalize: "收尾" },
category: {
timeout: "准备超时。",
runtime_disconnected: "Runtime 已断开连接。",
unconfirmed: "Runtime 未确认结果。",
rejected: "Runtime 拒绝了该操作。",
operation_failed: "Runtime 报告操作失败。",
core_error: "Core 无法读取、校验或记录初始化状态。",
core_restart: "Core 失去了初始化的所有权。",
compute_released: "沙箱已被释放或从未创建。",
} satisfies Record<ProvisioningCategory, string>,
initializationTitle: "初始化",
unavailable: "失败详情不可用",
unavailableHelp: "诊断读取失败或与当前状态不一致,请刷新后重试。",
checking: "正在读取失败详情",
Expand Down
3 changes: 3 additions & 0 deletions apps/web/src/lib/diagnostic-failure.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ export function diagnosticReason(failure: DiagnosticFailure, t: TFunction<"diagn
if (failure.params.index !== null) message += gap + t("setupIndex", { number: failure.params.index + 1 });
if (failure.params.exit_code !== null) message += gap + t("exitCode", { code: failure.params.exit_code });
}
if (failure.code === "environment_provisioning_failed" && "category" in failure.params && failure.params.category) {
message += gap + t(`category.${failure.params.category}`);
}
const status = connectionStatus(failure);
if (status !== null) message += gap + t("httpStatus", { status });
return message;
Expand Down
2 changes: 1 addition & 1 deletion contracts/agents-api/core-errors.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ The [Session diagnostics](./session-diagnostics.md) routes return these categori
| `internal_error` | Unknown or malformed outcome; no raw value is returned |
| `environment_connection_timeout` | Initial input connection deadline expired |
| `environment_unavailable` | Environment unavailable for initial input |
| `environment_provisioning_failed` | Hosted provisioning failure; params contain nullable `step`, `index`, `exit_code` from a sanitized receipt |
| `environment_provisioning_failed` | Hosted provisioning failure; params contain nullable `step`, `index`, `exit_code` from a sanitized receipt, and `category` for a failure without a confirmed step |

A database failure is an error, never an empty or healthy snapshot. Provisioning reasons and native messages are never parsed for categories or parameters.

Expand Down
29 changes: 29 additions & 0 deletions contracts/agents-api/core.openapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,29 @@ definitions:
- object
- updated_at
type: object
api.InitializationStepTiming:
properties:
duration_ms:
type: integer
index:
type: integer
x-nullable: true
step:
enum:
- file
- skill
- plugin
- configure
- npm
- python
- setup
- finalize
type: string
required:
- duration_ms
- index
- step
type: object
api.Installation:
properties:
address_bindings:
Expand Down Expand Up @@ -855,6 +878,11 @@ definitions:
allOf:
- $ref: '#/definitions/api.SessionDiagnosticFailure'
x-nullable: true
initialization_steps:
description: InitializationSteps lists the operations the Environment's initialization ran, in order.
items:
$ref: '#/definitions/api.InitializationStepTiming'
type: array
object:
enum:
- core.session_diagnostics
Expand All @@ -870,6 +898,7 @@ definitions:
type: string
required:
- failure
- initialization_steps
- object
- session_id
- status
Expand Down
17 changes: 16 additions & 1 deletion contracts/agents-api/session-diagnostics.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ Missing, malformed, foreign and deleted resources follow the Session and Turn no

## Session snapshot

The object is `core.session_diagnostics`, with `session_id`, the official Session `status`, and nullable `failure`. Each Session or Turn read uses one repeatable-read database snapshot and the public status projection, within a five-second budget that an earlier caller deadline shortens. The transaction is closed after the read, including on cancellation. `failure` is null unless the projection says `failed`. Its fields are:
The object is `core.session_diagnostics`, with `session_id`, the official Session `status`, nullable `failure` and `initialization_steps`. Each Session or Turn read uses one repeatable-read database snapshot and the public status projection, within a five-second budget that an earlier caller deadline shortens. The transaction is closed after the read, including on cancellation. `failure` is null unless the projection says `failed`. Its fields are:

| Field | Value |
| --- | --- |
Expand All @@ -27,6 +27,21 @@ A hosted provisioning failure takes precedence over input activity, and input ac

Provisioning parameters come from the confirmed structured receipt, persisted in the failure transaction that settles the Environment and its input and records the events. `step` is `setup`, `python`, `npm`, `file`, `skill`, `harness` or null. `index` is a nonnegative JSON-safe integer for setup only, otherwise null. `exit_code` is 1 to 255 for script steps, otherwise null. Details that were not recorded remain null. These private fields do not change the public failure reason or SSE.

`category` is set only when `step` is null, for a failure whose public reason is the generic `initialization did not complete`, and is otherwise null:

| `category` | Cause |
| --- | --- |
| `timeout` | The initialization budget or a transfer budget expired |
| `runtime_disconnected` | The Runtime connection closed, changed or was not registered |
| `unconfirmed` | An operation's effect is unknown, or its receipt is missing or invalid |
| `rejected` | The Runtime refused an operation before applying it |
| `operation_failed` | The Runtime confirmed a failed operation that has no public step label, such as a Plugin or snapshot finalization |
| `core_error` | Core failed to read, validate or record initialization state |
| `core_restart` | Core lost ownership of the initialization, by restart or lease loss |
| `compute_released` | The sandbox was released or never created before initialization completed; never set for an Environment whose initialization completed |

`initialization_steps` lists the operations the Environment's initialization ran, in order, from the same snapshot and for any Session status. Each entry has `step` (`file`, `skill`, `plugin`, `configure`, `npm`, `python`, `setup` or `finalize`), `index` (the setup command position for `setup`, otherwise null) and `duration_ms`, the Runtime-measured application time from the [Core–Runtime protocol](../../docs/runtime-protocol.md). Only operations whose result the Runtime reported with a `completed` or `failed` outcome appear, so a preparation-cache hit lists only `finalize`, and an unconfirmed operation is absent. Core also writes one `Environment initialization step` log line per entry with the Session, Environment, `step`, `index` and `duration_ms`. Core stores the list when the initialization ends, successfully or not, and logs each step as it finishes. The list is empty before any operation ran.

## Turn snapshot and Item receipt timing

The object is `core.turn_diagnostics`, with `session_id`, `turn_id`, the official root Turn `status`, nullable `failure`, `items` and `items_truncated`. A Turn failure has `code`, `params` and nullable `failed_at`, without a source or Turn ID. Only failed Turns have failure details; cancelled and completed Turns never inherit a classification from a private outcome.
Expand Down
Loading
Loading