feat(onlion): the private status says which endpoint and vector space are live

onlion/backend/monitoring.mjs

@@ -123,6 +123,17 @@ async function persistFailure(bucket, entry) {
     if (saved) return;
   }
 }
+/** Host of the configured embedding endpoint, or null. Never the key, never the path:
+ *  the host is what tells two deployments apart. */
+function embeddingHost(env) {
+  const raw = env?.NMG_EMBED_BASE_URL;
+  if (!raw) return null;
+  try {
+    return new URL(raw).host;
+  } catch {
+    return "unparseable";
+  }
+}
 export function reportFailure(error, status, kind = "request_failure") {
   const context = requests.getStore();
   if (!context) return;
@@ -137,6 +148,10 @@ export function reportFailure(error, status, kind = "request_failure") {
     ...classifyError(error),
     ...(context.user_id_hash ? { user_id_hash: context.user_id_hash } : {}),
     ...(context.request_id_hash ? { request_id_hash: context.request_id_hash } : {}),
+    // Which provider deployment the failing request was talking to. Without it every
+    // embedding incident had to be told apart by timestamp instead of by configuration.
+    ...(embeddingHost(context.env) ? { embedding_host: embeddingHost(context.env) } : {}),
+    ...(context.env.NMG_EMBED_MODEL ? { embedding_model: context.env.NMG_EMBED_MODEL } : {}),
     // NMG_DEBUG_ERRORS=1 surfaces the exception name/message for diagnosis; the
     // default record still carries no exception text, stack or request content.
     ...(context.env.NMG_DEBUG_ERRORS === "1"
@@ -149,6 +164,11 @@ export function reportFailure(error, status, kind = "request_failure") {
   localFailures.length = Math.min(localFailures.length, MAX_ERRORS);
   // No exception text, stack, request body, source statements or credentials.
   console.error(JSON.stringify({ event: "nmg_failure", ...entry }));
+  // Sharing the history across isolates costs two bucket round trips per failure - a read and
+  // a CAS write on one shared key - which is paid exactly when the service is already in
+  // trouble. The platform log carries the same entry for free and is where incidents are
+  // actually read, so the shared copy is opt-in.
+  if (context.env.NMG_ERROR_HISTORY !== "1") return;
   if (context.env.MEMORY && context.execution?.waitUntil) {
     context.execution.waitUntil(
       persistFailure(context.env.MEMORY, entry).catch(() =>

onlion/backend/worker.mjs

@@ -1678,6 +1678,62 @@ function overallServiceStatus({ blocked, maintenance, recent, history, queueStat
     ? "degraded"
     : "ok";
 }
+/** Bounded count of a field across failure entries, largest first. */
+function countTop(entries, keyOf, limit = 8) {
+  const counts = new Map();
+  for (const entry of entries) {
+    const key = keyOf(entry);
+    counts.set(key, (counts.get(key) ?? 0) + 1);
+  }
+  return [...counts.entries()]
+    .sort((a, b) => b[1] - a[1] || String(a[0]).localeCompare(String(b[0])))
+    .slice(0, limit)
+    .map(([key, count]) => ({ key, count }));
+}
+/** What the deployment is actually configured to talk to, for an operator looking at one
+ *  line of JSON.
+ *
+ *  The endpoint is reported as scheme, host and path - never the key - because "which endpoint
+ *  is this build calling" kept being a guess: a stale env var, a console setting the CLI deploy
+ *  did not carry, or a switch that was applied to the wrong environment all looked identical
+ *  from the outside. Values come from the client the requests themselves use, so they cannot
+ *  drift from what is in effect. */
+function embeddingDiagnostics(env) {
+  const raw = env.NMG_EMBED_BASE_URL ?? null;
+  let endpoint = null;
+  if (raw) {
+    try {
+      const url = new URL(raw);
+      endpoint = `${url.protocol}//${url.host}${url.pathname.replace(/\/+$/, "")}`;
+    } catch {
+      endpoint = "unparseable";
+    }
+  }
+  let client;
+  try {
+    client = createEmbeddingClientFromEnv({
+      ...env,
+      NMG_EMBED_TIMEOUT_MS: env.NMG_EMBED_TIMEOUT_MS ?? String(EMBED_TIMEOUT_MS),
+    });
+  } catch {
+    client = undefined;
+  }
+  const number = (value, fallback) => (Number(value) > 0 ? Number(value) : fallback);
+  return {
+    configured: !!(env.NMG_EMBED_PROVIDER || raw),
+    endpoint,
+    provider: env.NMG_EMBED_PROVIDER ?? (raw ? "openai" : null),
+    model: client?.model ?? env.NMG_EMBED_MODEL ?? null,
+    dimensions: client?.dimensions ?? number(env.NMG_EMBED_DIMENSIONS, null),
+    profile: client?.profile ?? null,
+    index_id: client?.indexId ?? null,
+    timeout_ms: number(env.NMG_EMBED_TIMEOUT_MS, EMBED_TIMEOUT_MS),
+    document_timeout_ms: number(env.NMG_EMBED_TIMEOUT_MS, EMBED_DOCUMENT_TIMEOUT_MS),
+    query_budget_ms: number(env.NMG_EMBED_QUERY_BUDGET_MS, EMBED_QUERY_BUDGET_MS),
+    write_budget_ms: number(env.NMG_EMBED_WRITE_BUDGET_MS, EMBED_WRITE_BUDGET_MS),
+    last_observation: embeddingObservation(),
+  };
+}
 async function adminStatusResponse(env) {
   const [sqlite, r2, reclaim, history] = await Promise.all([
     check(() => sqliteReady().then(() => true)),
@@ -1711,19 +1767,29 @@ async function adminStatusResponse(env) {
     },
     authentication: { configured: !!env.NMG_API_KEY },
     maintenance: reclaim.value ?? null,
-    embedding: {
-      configured: !!(env.NMG_EMBED_PROVIDER || env.NMG_EMBED_BASE_URL),
-      model: env.NMG_EMBED_MODEL || null,
-      status: "not_probed",
-      last_observation: embeddingObservation(),
-    },
+    revision: SOURCE_REVISION,
+    embedding: embeddingDiagnostics(env),
     checks_ms: { sqlite: sqlite.ms, r2: r2.ms },
     load,
     queue: queueState,
     // On demand, per instance: the same fields the slow-request report carries, so memory
     // can be watched while a deployment is live instead of only after an incident.
     memory: memFields({ sqlite: sqliteStats() }),
-    failures: { scope: "shared_best_effort", retention_hours: 24, maximum: 20, entries: failures },
+    failures: {
+      scope: env.NMG_ERROR_HISTORY === "1" ? "shared_best_effort" : "isolate_only",
+      retention_hours: 24,
+      maximum: 20,
+      // Reading twenty entries to answer "what is failing" is what made three incidents slow
+      // to diagnose, so the counts are computed here: same data, one glance.
+      summary: {
+        total: failures.length,
+        by_code: countTop(failures, (entry) => entry.code ?? "unknown"),
+        by_route: countTop(failures, (entry) => entry.route ?? "unknown"),
+        from: failures.at(-1)?.at ?? null,
+        to: failures[0]?.at ?? null,
+      },
+      entries: failures,
+    },
     limitations: [
       "Basic probes do not guarantee model availability.",
       "Queue and last embedding observation describe only this isolate.",

onlion/scripts/test-add-log-mode.mjs

@@ -341,6 +341,23 @@ try {
     );
   }
 
+  // With a provider configured, the private status answers which endpoint and which vector
+  // space are live - the question that had to be guessed three times during the 2026-10-29
+  // incidents.
+  {
+    const status = await call("/admin/status", undefined, {
+      "oai-authenticated-user-email": "owner@example.test",
+    });
+    assert.equal(status.status, 200, JSON.stringify(status.body));
+    assert.equal(status.body.embedding.configured, true);
+    assert.equal(status.body.embedding.endpoint, "https://embed.test/v1");
+    assert.equal(status.body.embedding.model, "bge-small-en-v1.5");
+    assert.match(status.body.embedding.index_id, /^.+@[0-9a-f]{12}$/);
+    assert.equal(status.body.embedding.dimensions, DIMENSIONS);
+    assert.ok(!JSON.stringify(status.body).includes("test-key"));
+    console.log("PASS: the private status reports the live endpoint, model and index identity.");
+  }
+
   // Reset drains the log as well as the partitions.
   const reset = await call("/reset", {
     user_id: "alice",

onlion/scripts/test-backend.mjs

@@ -533,6 +533,37 @@ print(json.dumps(r))`,
     "oai-authenticated-user-email": "owner@example.test",
     origin: "http://localhost",
   };
+  // The private status endpoint has to answer "what is this build actually talking to" without
+  // a guess: endpoint, model, index identity, the timings that bound a request. The key must
+  // never appear in it.
+  {
+    const signedOut = await call("a", "/admin/status");
+    assert.equal(signedOut.status, 401, "status is owner-only");
+    const ownerHeaders = { "oai-authenticated-user-email": "owner@example.test" };
+    const status = await call("a", "/admin/status", undefined, undefined, ownerHeaders);
+    assert.equal(status.status, 200, JSON.stringify(status.body));
+    const embedding = status.body.embedding;
+    // This suite runs without a provider, so the honest answer is "not configured" - and the
+    // timings still report the code defaults, because they bound a request either way.
+    assert.equal(embedding.configured, false);
+    assert.equal(embedding.endpoint, null);
+    assert.equal(embedding.index_id, null);
+    assert.equal(embedding.timeout_ms, 10_000);
+    assert.equal(embedding.document_timeout_ms, 25_000);
+    assert.equal(embedding.write_budget_ms, 45_000);
+    assert.ok(String(status.body.revision).length > 0, "the build identifies itself");
+    // workerd reports zeros here; the shape is what this asserts. The deployed runtime
+    // returns real numbers, which is how the 2026-10-29 numbers were finally read.
+    for (const key of ["rss", "heapUsed", "external", "arrayBuffers"])
+      assert.equal(typeof status.body.memory.mem[key], "number", `memory.${key} is reported`);
+    assert.equal(status.body.memory.instance.seq, 4, "the instance counts its requests");
+    assert.equal(status.body.failures.scope, "isolate_only", "shared history is opt-in");
+    assert.ok(Array.isArray(status.body.failures.summary.by_code));
+    const body = JSON.stringify(status.body);
+    assert.ok(!body.includes("test-key"), "the diagnostics must not echo the key");
+    assert.ok(!body.includes("api_key"), "the diagnostics must not carry a key field");
+  }
+
   const serviceHealth = await call("a", "/health");
   assert.equal(serviceHealth.body.memory_service, "daemon-core-aml-v1");
   assert.equal(serviceHealth.body.harness, false);