diff --git a/onlion/backend/monitoring.mjs b/onlion/backend/monitoring.mjs index b9e0c71b..76d598e2 100644 --- a/onlion/backend/monitoring.mjs +++ b/onlion/backend/monitoring.mjs @@ -123,6 +123,17 @@ async function persistFailure(bucket, entry) { if (saved) return; } } +/** Host of the configured embedding endpoint, or null. Never the key, never the path: + * the host is what tells two deployments apart. */ +function embeddingHost(env) { + const raw = env?.NMG_EMBED_BASE_URL; + if (!raw) return null; + try { + return new URL(raw).host; + } catch { + return "unparseable"; + } +} export function reportFailure(error, status, kind = "request_failure") { const context = requests.getStore(); if (!context) return; @@ -137,6 +148,10 @@ export function reportFailure(error, status, kind = "request_failure") { ...classifyError(error), ...(context.user_id_hash ? { user_id_hash: context.user_id_hash } : {}), ...(context.request_id_hash ? { request_id_hash: context.request_id_hash } : {}), + // Which provider deployment the failing request was talking to. Without it every + // embedding incident had to be told apart by timestamp instead of by configuration. + ...(embeddingHost(context.env) ? { embedding_host: embeddingHost(context.env) } : {}), + ...(context.env.NMG_EMBED_MODEL ? { embedding_model: context.env.NMG_EMBED_MODEL } : {}), // NMG_DEBUG_ERRORS=1 surfaces the exception name/message for diagnosis; the // default record still carries no exception text, stack or request content. ...(context.env.NMG_DEBUG_ERRORS === "1" @@ -149,6 +164,11 @@ export function reportFailure(error, status, kind = "request_failure") { localFailures.length = Math.min(localFailures.length, MAX_ERRORS); // No exception text, stack, request body, source statements or credentials. console.error(JSON.stringify({ event: "nmg_failure", ...entry })); + // Sharing the history across isolates costs two bucket round trips per failure - a read and + // a CAS write on one shared key - which is paid exactly when the service is already in + // trouble. The platform log carries the same entry for free and is where incidents are + // actually read, so the shared copy is opt-in. + if (context.env.NMG_ERROR_HISTORY !== "1") return; if (context.env.MEMORY && context.execution?.waitUntil) { context.execution.waitUntil( persistFailure(context.env.MEMORY, entry).catch(() => diff --git a/onlion/backend/worker.mjs b/onlion/backend/worker.mjs index 96bf3b56..185b6157 100644 --- a/onlion/backend/worker.mjs +++ b/onlion/backend/worker.mjs @@ -1678,6 +1678,62 @@ function overallServiceStatus({ blocked, maintenance, recent, history, queueStat ? "degraded" : "ok"; } +/** Bounded count of a field across failure entries, largest first. */ +function countTop(entries, keyOf, limit = 8) { + const counts = new Map(); + for (const entry of entries) { + const key = keyOf(entry); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return [...counts.entries()] + .sort((a, b) => b[1] - a[1] || String(a[0]).localeCompare(String(b[0]))) + .slice(0, limit) + .map(([key, count]) => ({ key, count })); +} +/** What the deployment is actually configured to talk to, for an operator looking at one + * line of JSON. + * + * The endpoint is reported as scheme, host and path - never the key - because "which endpoint + * is this build calling" kept being a guess: a stale env var, a console setting the CLI deploy + * did not carry, or a switch that was applied to the wrong environment all looked identical + * from the outside. Values come from the client the requests themselves use, so they cannot + * drift from what is in effect. */ +function embeddingDiagnostics(env) { + const raw = env.NMG_EMBED_BASE_URL ?? null; + let endpoint = null; + if (raw) { + try { + const url = new URL(raw); + endpoint = `${url.protocol}//${url.host}${url.pathname.replace(/\/+$/, "")}`; + } catch { + endpoint = "unparseable"; + } + } + let client; + try { + client = createEmbeddingClientFromEnv({ + ...env, + NMG_EMBED_TIMEOUT_MS: env.NMG_EMBED_TIMEOUT_MS ?? String(EMBED_TIMEOUT_MS), + }); + } catch { + client = undefined; + } + const number = (value, fallback) => (Number(value) > 0 ? Number(value) : fallback); + return { + configured: !!(env.NMG_EMBED_PROVIDER || raw), + endpoint, + provider: env.NMG_EMBED_PROVIDER ?? (raw ? "openai" : null), + model: client?.model ?? env.NMG_EMBED_MODEL ?? null, + dimensions: client?.dimensions ?? number(env.NMG_EMBED_DIMENSIONS, null), + profile: client?.profile ?? null, + index_id: client?.indexId ?? null, + timeout_ms: number(env.NMG_EMBED_TIMEOUT_MS, EMBED_TIMEOUT_MS), + document_timeout_ms: number(env.NMG_EMBED_TIMEOUT_MS, EMBED_DOCUMENT_TIMEOUT_MS), + query_budget_ms: number(env.NMG_EMBED_QUERY_BUDGET_MS, EMBED_QUERY_BUDGET_MS), + write_budget_ms: number(env.NMG_EMBED_WRITE_BUDGET_MS, EMBED_WRITE_BUDGET_MS), + last_observation: embeddingObservation(), + }; +} async function adminStatusResponse(env) { const [sqlite, r2, reclaim, history] = await Promise.all([ check(() => sqliteReady().then(() => true)), @@ -1711,19 +1767,29 @@ async function adminStatusResponse(env) { }, authentication: { configured: !!env.NMG_API_KEY }, maintenance: reclaim.value ?? null, - embedding: { - configured: !!(env.NMG_EMBED_PROVIDER || env.NMG_EMBED_BASE_URL), - model: env.NMG_EMBED_MODEL || null, - status: "not_probed", - last_observation: embeddingObservation(), - }, + revision: SOURCE_REVISION, + embedding: embeddingDiagnostics(env), checks_ms: { sqlite: sqlite.ms, r2: r2.ms }, load, queue: queueState, // On demand, per instance: the same fields the slow-request report carries, so memory // can be watched while a deployment is live instead of only after an incident. memory: memFields({ sqlite: sqliteStats() }), - failures: { scope: "shared_best_effort", retention_hours: 24, maximum: 20, entries: failures }, + failures: { + scope: env.NMG_ERROR_HISTORY === "1" ? "shared_best_effort" : "isolate_only", + retention_hours: 24, + maximum: 20, + // Reading twenty entries to answer "what is failing" is what made three incidents slow + // to diagnose, so the counts are computed here: same data, one glance. + summary: { + total: failures.length, + by_code: countTop(failures, (entry) => entry.code ?? "unknown"), + by_route: countTop(failures, (entry) => entry.route ?? "unknown"), + from: failures.at(-1)?.at ?? null, + to: failures[0]?.at ?? null, + }, + entries: failures, + }, limitations: [ "Basic probes do not guarantee model availability.", "Queue and last embedding observation describe only this isolate.", diff --git a/onlion/scripts/test-add-log-mode.mjs b/onlion/scripts/test-add-log-mode.mjs index a5feb548..5207899b 100644 --- a/onlion/scripts/test-add-log-mode.mjs +++ b/onlion/scripts/test-add-log-mode.mjs @@ -341,6 +341,23 @@ try { ); } + // With a provider configured, the private status answers which endpoint and which vector + // space are live - the question that had to be guessed three times during the 2026-10-29 + // incidents. + { + const status = await call("/admin/status", undefined, { + "oai-authenticated-user-email": "owner@example.test", + }); + assert.equal(status.status, 200, JSON.stringify(status.body)); + assert.equal(status.body.embedding.configured, true); + assert.equal(status.body.embedding.endpoint, "https://embed.test/v1"); + assert.equal(status.body.embedding.model, "bge-small-en-v1.5"); + assert.match(status.body.embedding.index_id, /^.+@[0-9a-f]{12}$/); + assert.equal(status.body.embedding.dimensions, DIMENSIONS); + assert.ok(!JSON.stringify(status.body).includes("test-key")); + console.log("PASS: the private status reports the live endpoint, model and index identity."); + } + // Reset drains the log as well as the partitions. const reset = await call("/reset", { user_id: "alice", diff --git a/onlion/scripts/test-backend.mjs b/onlion/scripts/test-backend.mjs index f665e517..a18c93d6 100644 --- a/onlion/scripts/test-backend.mjs +++ b/onlion/scripts/test-backend.mjs @@ -533,6 +533,37 @@ print(json.dumps(r))`, "oai-authenticated-user-email": "owner@example.test", origin: "http://localhost", }; + // The private status endpoint has to answer "what is this build actually talking to" without + // a guess: endpoint, model, index identity, the timings that bound a request. The key must + // never appear in it. + { + const signedOut = await call("a", "/admin/status"); + assert.equal(signedOut.status, 401, "status is owner-only"); + const ownerHeaders = { "oai-authenticated-user-email": "owner@example.test" }; + const status = await call("a", "/admin/status", undefined, undefined, ownerHeaders); + assert.equal(status.status, 200, JSON.stringify(status.body)); + const embedding = status.body.embedding; + // This suite runs without a provider, so the honest answer is "not configured" - and the + // timings still report the code defaults, because they bound a request either way. + assert.equal(embedding.configured, false); + assert.equal(embedding.endpoint, null); + assert.equal(embedding.index_id, null); + assert.equal(embedding.timeout_ms, 10_000); + assert.equal(embedding.document_timeout_ms, 25_000); + assert.equal(embedding.write_budget_ms, 45_000); + assert.ok(String(status.body.revision).length > 0, "the build identifies itself"); + // workerd reports zeros here; the shape is what this asserts. The deployed runtime + // returns real numbers, which is how the 2026-10-29 numbers were finally read. + for (const key of ["rss", "heapUsed", "external", "arrayBuffers"]) + assert.equal(typeof status.body.memory.mem[key], "number", `memory.${key} is reported`); + assert.equal(status.body.memory.instance.seq, 4, "the instance counts its requests"); + assert.equal(status.body.failures.scope, "isolate_only", "shared history is opt-in"); + assert.ok(Array.isArray(status.body.failures.summary.by_code)); + const body = JSON.stringify(status.body); + assert.ok(!body.includes("test-key"), "the diagnostics must not echo the key"); + assert.ok(!body.includes("api_key"), "the diagnostics must not carry a key field"); + } + const serviceHealth = await call("a", "/health"); assert.equal(serviceHealth.body.memory_service, "daemon-core-aml-v1"); assert.equal(serviceHealth.body.harness, false);