From bf388e34db5d4e7cbfba7b7d1844020e209c0151 Mon Sep 17 00:00:00 2001 From: Shuby Mishra Date: Fri, 20 Nov 2026 21:39:43 +0000 Subject: [PATCH] feat(observability): add pure-JS cluster dashboard UI at /dashboard Signed-off-by: Shuby Mishra --- docs/configuration.md | 5 +- docs/dashboard.md | 54 ++ pyproject.toml | 3 + src/server/observability/router.py | 12 + src/server/observability/static/activity.js | 166 +++++++ src/server/observability/static/app.js | 296 +++++++++++ src/server/observability/static/charts.js | 212 ++++++++ src/server/observability/static/dom.js | 73 +++ src/server/observability/static/format.js | 30 ++ src/server/observability/static/index.html | 12 + src/server/observability/static/node-time.js | 162 ++++++ src/server/observability/static/state.js | 231 +++++++++ src/server/observability/static/style.css | 351 +++++++++++++ src/server/observability/static/views.js | 461 ++++++++++++++++++ .../observability/static/visualization.js | 179 +++++++ tests/test_observability.py | 18 + 16 files changed, 2263 insertions(+), 2 deletions(-) create mode 100644 docs/dashboard.md create mode 100644 src/server/observability/static/activity.js create mode 100644 src/server/observability/static/app.js create mode 100644 src/server/observability/static/charts.js create mode 100644 src/server/observability/static/dom.js create mode 100644 src/server/observability/static/format.js create mode 100644 src/server/observability/static/index.html create mode 100644 src/server/observability/static/node-time.js create mode 100644 src/server/observability/static/state.js create mode 100644 src/server/observability/static/style.css create mode 100644 src/server/observability/static/views.js create mode 100644 src/server/observability/static/visualization.js diff --git a/docs/configuration.md b/docs/configuration.md index e239e3d63..ac7ed96a7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -127,8 +127,9 @@ for. On one machine every pool has one replica, in process. See ## Inspection API The API server serves a read-only inventory of runs at -`/api/v1/inspect/inventory` and a run's logs at -`/api/v1/inspect/logs?run_id=`. `/api/v1/inspect/activity` shows which +`/api/v1/inspect/inventory`, a run's logs at +`/api/v1/inspect/logs?run_id=`, and the [cluster dashboard](dashboard.md) at +`/dashboard`. `/api/v1/inspect/activity` shows which `Workload`s held which GPUs in a time window, and `/api/v1/inspect/query` reads those GPUs' DCGM metrics. See `/docs` for their fields. diff --git a/docs/dashboard.md b/docs/dashboard.md new file mode 100644 index 000000000..f0bb7a4af --- /dev/null +++ b/docs/dashboard.md @@ -0,0 +1,54 @@ +# Cluster dashboard + +`/dashboard` is an operator UI served by the API server over the read-only +inspection API under `/api/v1/inspect`. It is plain ES-module JavaScript with +no build step; agents use the same JSON routes (`/docs` and `/openapi.json` +describe their fields). + +## Run it + +```sh +make server +``` + +Then open `http://127.0.0.1:9003/dashboard`. + +On Kubernetes, the API server already serves `/dashboard`, so no separate +deployment is needed: + +```sh +kubectl -n openrl-system port-forward svc/open-rl-api-server-service 8080:8000 +``` + +Then open `http://127.0.0.1:8080/dashboard`. + +Logs need `OPEN_RL_LOG_STORE`, nodes and pods need Kubernetes, and GPU duty +charts need `OPEN_RL_PROMETHEUS_URL`. A source that cannot be read shows as an +error, never as an empty cluster. See +[Inspection API](configuration.md#inspection-api) for the environment variables +and IAM permissions. + +## Pages + +- **Overview** lists runs with status, completed steps, elapsed time, search, + and status filters. +- **Run** shows status, settings, an activity timeline with GPU rows and duty, + trainer metrics and operation durations by step, and worker or client logs. + Click the timeline to open logs around that time. +- **Nodes** shows which process held which GPU and when, each GPU's duty cycle, + and GPU utilization and memory. Drag or press A/D to pan; use Ctrl/⌘ + wheel, + pinch, or W/S to zoom. +- **Health** lists pod and node problems and sources that could not be read. + +## Lifecycle + +Each run in `/api/v1/inspect/inventory` has a `lifecycle`, which says where it +is: + +- **ended**: the API server marked it finished, its workers stopped, or it has + no workers and no live session. +- **failed**: one of its workers is in a failed state. +- **waiting**: a worker is not placed yet, or the server has not created one + for a live session. +- **placed**: every worker is placed. +- **active**: the scheduler cannot be read, so placement is unknown. diff --git a/pyproject.toml b/pyproject.toml index 644a9a099..0b3c112d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,9 @@ dev = [ where = ["src"] include = ["server*", "training*", "accel_timeslicer*"] +[tool.setuptools.package-data] +"server.observability" = ["static/*"] + [tool.uv] override-dependencies = [ # vLLM still carries an older transformers constraint; keep the project on diff --git a/src/server/observability/router.py b/src/server/observability/router.py index 88203f73e..7b78eab74 100644 --- a/src/server/observability/router.py +++ b/src/server/observability/router.py @@ -2,10 +2,12 @@ import os import time from datetime import datetime +from pathlib import Path from typing import Annotated, Literal import httpx from fastapi import APIRouter, HTTPException, Query +from fastapi.responses import FileResponse from server.model_metadata import decode_model_metadata from server.observability import cloud_logging, facts, fleet, metrics @@ -17,6 +19,7 @@ from server.worker_manager import owner_id router = APIRouter(tags=["inspection"]) +STATIC = Path(__file__).parent / "static" BOUNDS = "ISO-8601 or Unix seconds; since also takes a span before until, e.g. 1h. Default: the 30 minutes ending now; at most 7 days." Bound = Annotated[str | None, Query(max_length=64, description=BOUNDS)] CURSOR_EXPIRED = "Invalid or expired log cursor; repeat the query without it" @@ -24,6 +27,15 @@ LOG_STORES = {"cloud-logging": cloud_logging.log_page} +@router.get("/dashboard", include_in_schema=False) +@router.get("/dashboard/", include_in_schema=False) +@router.get("/dashboard/assets/{name}", include_in_schema=False) +async def dashboard(name: str = "index.html"): + if (path := (STATIC / name).resolve()).parent != STATIC.resolve() or not path.is_file(): + raise HTTPException(404, "Not found") + return FileResponse(path, headers={"Cache-Control": "no-store"}) + + @router.get("/api/v1/inspect/inventory") async def inventory(): """Runs with their lifecycle (ended, failed, waiting, placed, active), pods and nodes; the cluster's nodes with their GPUs diff --git a/src/server/observability/static/activity.js b/src/server/observability/static/activity.js new file mode 100644 index 000000000..324936a5b --- /dev/null +++ b/src/server/observability/static/activity.js @@ -0,0 +1,166 @@ +// GPU & process activity timelines (Run across-nodes + per-GPU Duty cycle, and Node activity). +import { renderChart } from "./charts.js"; +import { escape } from "./format.js"; +import { inspectUrl, nodeLink, setCrosshair, ui, use, viewLink } from "./state.js"; +import { DCGM_READING_SECONDS, activitySource, adaptLanes, axisTime, chartTime, deviceLabel, dutyBands, opLabel, opRank, placementColor, processLabel, shortNodeName, valueText } from "./visualization.js"; + +const trackRegistry = new Map(); +let trackSeq = 0; + +export function resetActivity() { + trackRegistry.clear(); + trackSeq = 0; +} + +function renderGpuDuty(node, device, range, runs) { + const bounds = { node_id: node, since: new Date(range.start * 1000).toISOString(), until: new Date(range.now * 1000).toISOString() }; + const opts = { poll: 15000, pollWhen: range.live, remember: !range.live }; + const utilization = use(inspectUrl("query", { name: "gpu_util", ...bounds }), { ...opts, scope: `gpu-util:${node}` }); + const holders = use(inspectUrl("activity", bounds), { ...opts, scope: `gpu-act:${node}` }); + const points = utilization.data?.series?.find((s) => s.labels.UUID === device.id)?.points || []; + return renderChart({ + id: `gpu-duty:${node}:${device.id}`, + title: `Duty cycle · ${device.label}`, + unit: "%", + points, + start: range.start, + end: range.now, + min: 0, + max: 100, + readingSeconds: DCGM_READING_SECONDS, + bands: dutyBands(adaptLanes(holders.data?.lanes || [], runs), device.id), + empty: utilization.error || (!utilization.data && "Loading…") || "No samples for this GPU", + }); +} + +export function renderActivity({ + placements = [], + range, + inventory, + acrossNodes = false, + activeOnly = false, + selected = null, + selectable = false, + error = null, + pending = false, + showSource = true, + back = null, + pickLogs = false, + duty = null, + slots = null, +}) { + const mode = acrossNodes ? ui.activityMode : "lanes"; + const activities = placements.toSorted((a, b) => + (acrossNodes ? (({ trainer: 0, sampler: 1 }[a.role] ?? 2) - ({ trainer: 0, sampler: 1 }[b.role] ?? 2) || String(a.node || "").localeCompare(String(b.node || ""))) : 0) || + a.start - b.start || + a.id.localeCompare(b.id)); + const shown = activeOnly ? activities.filter((p) => p.intervals.length || p.error) : activities; + + function gpuOf(p, id) { + const device = inventory.cluster.nodes.find((n) => n.name === p.node)?.devices.find((d) => d.id === id); + return { id, label: device ? `GPU ${deviceLabel(device.name)}` : slots?.[id] != null ? `GPU ${slots[id]}` : id.slice(0, 12) }; + } + + const rows = shown.flatMap((p) => (acrossNodes && p.devices?.length ? p.devices : [null]).map((id, i) => ({ + key: `${p.id}:${id ?? "none"}`, + lane: p, + device: id && gpuOf(p, id), + first: i === 0, + lead: i === 0 && p.devices.length > 1, + }))); + const warnings = [...new Set(activities.map((p) => p.errorStatus !== 404 && p.error).filter(Boolean))]; + const ops = [...new Set(shown.flatMap((p) => Object.keys(p.ops || {})))].sort((a, b) => opRank(a) - opRank(b) || a.localeCompare(b)); + const empty = error ? `Activity unavailable: ${error}` : pending ? "Loading activity…" : acrossNodes && !activities.length ? "No worker processes for this run in this window" : "No recorded activity in this window"; + const href = (p) => acrossNodes + ? inventory.cluster.nodes.some((n) => n.name === p.node) ? nodeLink(p.id, range) : null + : inventory.runs.some((r) => r.run_id === p.run_ids?.[0]) ? viewLink(`run/${encodeURIComponent(p.run_ids[0])}/activity`, { duration: range.now - range.start, end: range.now, back }) : null; + const blocks = (spans) => spans.map(([a, b]) => `M${1000 * (a - range.start) / (range.now - range.start)},0 H${1000 * (b - range.start) / (range.now - range.start)} V24 H${1000 * (a - range.start) / (range.now - range.start)} Z`).join(" "); + const rowLabel = (p) => acrossNodes ? `${p.label} · ${processLabel(p, inventory.cluster.nodes)} · ${p.node}` : p.label; + const dutyText = (device) => { const v = device && duty?.[device.id]; return Number.isFinite(v) ? `${v}%` : "—"; }; + + const reg = (spans, label, source) => { + const id = `tr-${trackSeq++}`; + trackRegistry.set(id, { spans, label, source, range, pickLogs }); + return id; + }; + + const head = acrossNodes + ? `

GPU activity

One row per accelerator. Colored bars are recorded work on that GPU; translucent bars are GPU turns or placements held. Duty is the share of time a kernel ran on the GPU, measured by DCGM.

+
` + : ""; + + let body; + if (!shown.length) { + body = `

${escape(empty)}

`; + } else { + const axis = [0, 1, 2, 3, 4].map((tick) => `${escape(axisTime(range.start + (range.now - range.start) * tick / 4, range))}`).join(""); + const rowsHtml = rows.map(({ key, lane: p, device, first, lead }) => { + const name = acrossNodes ? processLabel(p, inventory.cluster.nodes, { gpus: false }) : p.label; + const meta = acrossNodes ? p.node ? shortNodeName(p.node, inventory.cluster.nodes) : p.ended && p.pod ? p.pod : "Node unassigned" : p.role || "process"; + const link = href(p); + const status = p.error ? "Ops unavailable" : !p.intervals.length ? "No recorded activity" : ""; + const color = placementColor(p); + const isOpen = ui.openGpus.has(key); + const metaText = `${meta}${p.ended ? " · Process ended" : p.phase && p.phase !== "ready" ? ` · ${p.phase.replace("_", " ")}` : ""}${!link ? acrossNodes ? p.node ? " · Node no longer reported" : "" : p.run_ids?.length ? " · Run record unavailable" : " · No linked run" : ""}`; + const labelEl = first + ? `<${link ? `a href="${escape(link)}"` : 'span aria-disabled="true"'} class="activity-label ${color}" data-key="label:${escape(p.id)}" title="${escape(acrossNodes ? `${name} · ${p.node || "Node unassigned"}${p.node && !link ? " · Node no longer reported" : ""}` : p.label)}">${escape(name)}${escape(metaText)}` + : ""; + const gpuEl = acrossNodes + ? device && p.node + ? `` + : `${escape(device?.label ?? "GPUs not recorded")}` + : ""; + + const rl = rowLabel(p); + const waitPath = p.waits?.length ? `` : ""; + const selectAttrs = selectable ? ` role="button" tabindex="0" data-placement="${escape(p.id)}" data-selected="${p.id === selected}"` : ""; + const basePath = ``; + const opPaths = Object.entries(p.ops || {}).sort(([a], [b]) => opRank(a) - opRank(b)).map(([op, spans]) => + ``).join(""); + const svg = mode === "lanes" ? `${waitPath}${basePath}${opPaths}` : ""; + const statusHtml = status && first ? `${escape(status)}` : ""; + const dutyHtml = acrossNodes ? `${escape(dutyText(device))}` : ""; + const dutyRow = (isOpen || mode === "duty") && device && p.node ? `
${renderGpuDuty(p.node, device, range, inventory.runs)}
` : ""; + return `
${labelEl}${gpuEl}
${svg}${statusHtml}
${dutyHtml}
${dutyRow}`; + }).join(""); + + const legend = ops.length && mode === "lanes" + ? `
${ops.map((op) => `${escape(opLabel(op))}`).join("")}held, unlabelled${shown.some((p) => p.waits?.length) ? 'waiting for the GPU' : ""}
` + : ""; + body = `

${acrossNodes ? "Process" : "Run"}

${acrossNodes ? "

GPU

" : ""}
${axis}
${acrossNodes ? 'Duty' : ""}
${rowsHtml}${legend}
`; + } + + const errHtml = error && shown.length ? `

Activity unavailable: ${escape(error)}

` : ""; + const warnHtml = warnings.length ? `

${escape(warnings.join(" · "))}

` : ""; + const srcHtml = showSource && activities.length ? `

${escape(activitySource(activities))}

` : ""; + return `
${head}${body}${errHtml}${warnHtml}${srcHtml}
`; +} + +export function hoverActivity(section, event, range) { + const tip = section.querySelector(".activity-tooltip"); + if (!tip) return; + if (ui.navigating || event.buttons) { + tip.hidden = true; + setCrosshair(null, range); + return; + } + const track = event.target.closest(".activity-track"); + if (track) { + const box = track.getBoundingClientRect(); + setCrosshair(range.start + Math.max(0, Math.min(1, (event.clientX - box.left) / (box.width || 1))) * (range.now - range.start), range); + } + const block = event.target.closest(".activity-block[data-track-id]"); + const meta = block && trackRegistry.get(block.dataset.trackId); + if (!meta) { tip.hidden = true; return; } + const box = block.ownerSVGElement.getBoundingClientRect(); + const at = meta.range.start + (event.clientX - box.left) / (box.width || 1) * (meta.range.now - meta.range.start); + const interval = meta.spans.find(([a, b]) => at >= a && at <= b); + if (!interval) { tip.hidden = true; return; } + const [a, b] = interval, date = (v) => new Date(v * 1000).toISOString().slice(0, 10); + tip.querySelector(".activity-tooltip-label").textContent = meta.label; + tip.querySelector(".activity-tooltip-time").textContent = `${date(a)} · ${chartTime(a, true)} – ${date(a) === date(b) ? "" : `${date(b)} · `}${chartTime(b, true)}`; + tip.querySelector(".activity-tooltip-detail").textContent = `Visible interval · ${valueText(b - a, "s")}${meta.source ? ` · ${meta.source}` : ""}${meta.pickLogs ? " · Click for logs" : ""}`; + tip.hidden = false; + tip.style.left = `${Math.max(8, Math.min(event.clientX + 12, innerWidth - (tip.offsetWidth || 300) - 8))}px`; + tip.style.top = `${Math.max(8, Math.min(event.clientY + 12, innerHeight - (tip.offsetHeight || 85) - 8))}px`; +} diff --git a/src/server/observability/static/app.js b/src/server/observability/static/app.js new file mode 100644 index 000000000..fff1fe66f --- /dev/null +++ b/src/server/observability/static/app.js @@ -0,0 +1,296 @@ +// OpenRL cluster dashboard entrypoint: shell, inventory polling, routing, and event delegation. +import { hoverActivity, resetActivity } from "./activity.js"; +import { hoverChart, inspectChart, resetCharts } from "./charts.js"; +import { copyLink, morph } from "./dom.js"; +import { escape } from "./format.js"; +import { attachNodeTime } from "./node-time.js"; +import { beginRender, currentView, endRender, get, inspectUrl, now, parseView, rangeOf, retryQuery, saved, setCrosshair, syncHash, timeAt, ui, view, viewLink } from "./state.js"; +import { adaptInventory } from "./visualization.js"; +import { followLogs, loadLogs, logsState, pauseLogs, renderHealth, renderNodes, renderOverview, renderRun, stopLogs } from "./views.js"; + +const root = document.getElementById("app"); +let inventoryTimer = null; +let inventoryController = null; +let copyTimer = null; + +async function pollInventory() { + if (ui.inventory.pending) return; + clearTimeout(inventoryTimer); + inventoryController?.abort(); + const controller = inventoryController = new AbortController(); + ui.inventory.pending = true; + let retry = false; + try { + const { data, etag } = await get(inspectUrl("inventory"), controller.signal, ui.inventory.etag ? ui.inventory : null); + if (controller.signal.aborted) return; + ui.inventory.failures = 0; + ui.inventory.error = data.error || null; + ui.inventory.status = null; + ui.inventory.stale = false; + ui.inventory.etag = etag; + if (!ui.inventory.error || !ui.inventory.data) ui.inventory.data = adaptInventory(data); + } catch (error) { + if (controller.signal.aborted) return; + ui.inventory.failures += 1; + retry = (!error.status || error.status >= 500 || error.status === 429) && ui.inventory.failures < 4; + if (retry) ui.inventory.stale = Boolean(ui.inventory.data); + else { + ui.inventory.error = error.message; + ui.inventory.status = error.status || null; + ui.inventory.stale = false; + } + } finally { + if (!controller.signal.aborted) { + ui.inventory.pending = false; + inventoryController = null; + render(); + const delay = retry ? 2000 * 2 ** (ui.inventory.failures - 1) : 10000; + inventoryTimer = setTimeout(() => { if (!document.hidden) pollInventory(); }, delay); + } + } +} + +function render() { + syncHash(); + beginRender(); + resetCharts(); + resetActivity(); + + const inv = ui.inventory; + const section = view.page === "run" ? (view.back ? "nodes" : "overview") : view.page; + const connection = inv.error ? (inv.data ? "Stale snapshot" : "Unavailable") : inv.stale ? "Reconnecting…" : !inv.data ? "Connecting…" : inv.data.cluster.available ? "Connected" : "Cluster unavailable"; + const nav = [["overview", "Overview"], ["nodes", "Nodes"], ["health", "Health"]] + .map(([page, label]) => `${escape(label)}`).join(""); + + let main; + if (inv.data) { + if (view.page === "run") main = renderRun(inv.data); + else if (view.page === "nodes") { stopLogs(); main = renderNodes(inv.data); } + else { stopLogs(); main = view.page === "health" ? renderHealth(inv.data) : renderOverview(inv.data); } + } else { + main = inv.error ? "" : "Loading cluster…"; + } + + endRender(); + morph(root, `
+
OpenRL${escape(connection)}
+
${escape(inv.error || "")}${inv.error && inv.data ? escape(` Showing the snapshot from ${inv.data.observed_at}.`) : ""}
+
${main}
+
`); + + if (view.page === "nodes" && view.node && ui.scrollSelection) { + const detail = document.getElementById("placement-detail"); + if (detail) { ui.scrollSelection = false; detail.scrollIntoView({ block: "start" }); } + } + if (logsState.needsScroll) { + const lines = document.getElementById("log-lines"); + if (lines) { lines.scrollTop = lines.scrollHeight - lines.clientHeight - logsState.fromBottom; } + logsState.fromBottom = 0; + logsState.needsScroll = false; + } +} + +ui.render = render; + +function keepViewport() { + root.style.setProperty("min-height", `${window.scrollY + innerHeight}px`); +} +function closeNodeDetails() { + const prev = view.node; + view.node = view.placement = null; + root.style.removeProperty("min-height"); + render(); + if (prev) root.querySelector(`[data-node-details="${CSS.escape(prev)}"]`)?.focus({ preventScroll: true }); +} + +root.addEventListener("click", async (event) => { + const target = event.target; + if (target.closest("[data-retry-snapshot]")) { ui.inventory.failures = 0; pollInventory(); return; } + + const retryBtn = target.closest("[data-retry-query]"); + if (retryBtn) { retryQuery(retryBtn.dataset.retryQuery); return; } + + const filterBtn = target.closest("[data-run-filter]"); + if (filterBtn) { view.status = filterBtn.dataset.runFilter; render(); return; } + + const copyBtn = target.closest("[data-copy-view]"); + if (copyBtn) { + copyBtn.textContent = await copyLink(new URL(currentView(true), location.href).href); + clearTimeout(copyTimer); + copyTimer = setTimeout(() => { if (copyBtn.isConnected) copyBtn.textContent = "Copy link"; }, 2000); + return; + } + + const shiftBtn = target.closest("[data-time-shift]"); + if (shiftBtn) { + const observed = now(ui.inventory.data), range = rangeOf(ui.inventory.data, view), dur = range.now - range.start; + const end = range.now + Number(shiftBtn.dataset.timeShift) * dur; + view.end = end >= observed ? null : end; + render(); + return; + } + if (target.closest("[data-time-live]")) { view.end = null; render(); return; } + + const modeBtn = target.closest("[data-activity-mode]"); + if (modeBtn) { ui.activityMode = modeBtn.dataset.activityMode; render(); return; } + + const gpuToggle = target.closest("[data-toggle-gpu]"); + if (gpuToggle) { + const key = gpuToggle.dataset.toggleGpu; + if (ui.openGpus.has(key)) ui.openGpus.delete(key); + else ui.openGpus.add(key); + render(); + return; + } + + const chartModeBtn = target.closest("[data-chart-mode]"); + if (chartModeBtn) { ui.chartModes.set(chartModeBtn.dataset.chartMode, chartModeBtn.dataset.mode); render(); return; } + + const nodeBtn = target.closest("[data-node-details]"); + if (nodeBtn) { + const name = nodeBtn.dataset.nodeDetails; + if (view.node === name) closeNodeDetails(); + else { root.style.removeProperty("min-height"); view.node = name; view.placement = null; view.gpu = "all"; ui.scrollSelection = false; render(); } + return; + } + if (target.closest("[data-close-details]")) { closeNodeDetails(); return; } + + const devBtn = target.closest("[data-device]"); + if (devBtn) { keepViewport(); view.gpu = devBtn.dataset.device; render(); return; } + + const placementEl = target.closest("[data-placement]"); + if (placementEl && view.page === "nodes") { + const id = placementEl.dataset.placement, nodeName = placementEl.dataset.placementNode || view.node; + if (view.node !== nodeName) root.style.removeProperty("min-height"); + if (placementEl.closest(".capacity-allocation")) view.gpu = "all"; + view.node = nodeName; + view.placement = id; + render(); + return; + } + + const pickTrack = target.closest(".activity-track[data-pick-logs]"); + if (pickTrack && view.page === "run") { + const range = rangeOf(ui.inventory.data, view); + const box = pickTrack.getBoundingClientRect(); + const at = Math.round(range.start + Math.max(0, Math.min(1, (event.clientX - box.left) / (box.width || 1))) * (range.now - range.start)); + location.hash = viewLink(`run/${encodeURIComponent(view.id)}/logs`, { duration: view.duration, end: range.now, back: view.back, event: at, q: view.q, pod: view.pod }); + return; + } + + if (target.closest("[data-log-follow]")) { if (view.end === null) pauseLogs(); else followLogs(); return; } + if (target.closest("[data-log-retry]") && logsState.params) { loadLogs(logsState.params, logsState.failedCursor); return; } + const olderBtn = target.closest("[data-log-older]"); + if (olderBtn && logsState.params) { + const cursor = olderBtn.dataset.logOlder; + if (view.end === null) { + view.end = logsState.data?.windowEnd ?? logsState.rangeNow; + logsState.queryKey = `${logsState.filterKey}:${view.end}`; + if (logsState.data) logsState.data = { ...logsState.data, queryKey: logsState.queryKey }; + } + loadLogs(logsState.params, cursor); + } +}); + +root.addEventListener("input", (event) => { + const el = event.target; + if (el.id === "run-search" || el.id === "log-search") { view.q = el.value; render(); } +}); + +root.addEventListener("change", (event) => { + const el = event.target; + if (el.id === "log-source") { view.pod = el.value; render(); } + else if (el.matches("[data-log-everything]")) { logsState.everything = el.checked; render(); } + else if (el.matches("[data-time-duration]")) { + const range = rangeOf(ui.inventory.data, view); + view.duration = Number(el.value); + if (view.end !== null) view.end = range.now; + render(); + } else if (el.matches("[data-time-end]")) { + if (!el.validity.valid) { el.reportValidity(); return; } + const observed = now(ui.inventory.data); + const end = el.value ? Date.parse(`${el.value}Z`) / 1000 : null; + view.end = Number.isFinite(end) ? Math.min(end, observed) : null; + render(); + } +}); + +root.addEventListener("pointermove", (event) => { + const plot = event.target.closest(".chart-plot"); + if (plot) { hoverChart(plot, event); return; } + const section = event.target.closest("section[data-crosshair-host]"); + if (section) { hoverActivity(section, event, rangeOf(ui.inventory.data, view)); return; } + const track = event.target.closest(".capacity-track"); + if (track) setCrosshair(event.buttons ? null : timeAt({ currentTarget: track, clientX: event.clientX }, rangeOf(ui.inventory.data, view)), rangeOf(ui.inventory.data, view)); +}); + +root.addEventListener("pointerout", (event) => { + const plot = event.target.closest(".chart-plot"); + if (plot && !plot.contains(event.relatedTarget)) { + const hover = plot.querySelector(".chart-hover"); + if (hover) hover.hidden = true; + setCrosshair(null); + } + const section = event.target.closest("section[data-crosshair-host]"); + if (section && !section.contains(event.relatedTarget)) { + const tip = section.querySelector(".activity-tooltip"); + if (tip) tip.hidden = true; + setCrosshair(null); + } + const track = event.target.closest(".capacity-track"); + if (track && !track.contains(event.relatedTarget)) setCrosshair(null); +}); + +root.addEventListener("scroll", (event) => { + const box = event.target; + if (box.id === "log-lines" && view.end === null && box.scrollHeight - box.clientHeight - box.scrollTop > 24) pauseLogs(); +}, true); + +root.addEventListener("keydown", (event) => { + const plot = event.target.closest(".chart-plot"); + if (plot && inspectChart(plot, event.key)) { event.preventDefault(); event.stopPropagation(); return; } + const placementEl = event.target.closest("[data-placement]"); + if (placementEl && view.page === "nodes" && ["Enter", " "].includes(event.key)) { + event.preventDefault(); + view.placement = placementEl.dataset.placement; + render(); + } +}); + +window.addEventListener("keydown", (event) => { + if (event.key !== "Escape" || event.defaultPrevented) return; + const picker = root.querySelector("details.time-picker[open]"); + if (picker) { picker.open = false; picker.querySelector("summary")?.focus(); event.preventDefault(); return; } + if (view.page === "nodes" && (view.node || view.placement)) closeNodeDetails(); +}); + +window.addEventListener("click", (event) => { + const picker = root.querySelector("details.time-picker[open]"); + if (picker && !picker.contains(event.target)) picker.open = false; +}); + +window.addEventListener("scroll", () => { + for (const tip of root.querySelectorAll(".activity-tooltip")) tip.hidden = true; + setCrosshair(null); +}); +window.addEventListener("resize", () => { + for (const el of root.querySelectorAll(".activity-tooltip, .chart-hover")) el.hidden = true; + setCrosshair(null); +}); + +window.addEventListener("hashchange", () => { + const next = parseView(); + if (view.page !== next.page || view.id !== next.id) { + window.scrollTo(0, 0); + ui.scrollSelection = true; + root.style.removeProperty("min-height"); + } + Object.assign(view, next); + render(); +}); + +document.addEventListener("visibilitychange", () => { if (!document.hidden) pollInventory(); }); + +attachNodeTime(root); +render(); +pollInventory(); diff --git a/src/server/observability/static/charts.js b/src/server/observability/static/charts.js new file mode 100644 index 000000000..894226500 --- /dev/null +++ b/src/server/observability/static/charts.js @@ -0,0 +1,212 @@ +// Stretched SVG charts (Running / Readings / Duty 1 min modes, band clipPaths, pointer/keyboard inspection) and time picker. +import { escape } from "./format.js"; +import { MAX_WINDOW, WINDOWS, setCrosshair, ui } from "./state.js"; +import { axisTime, chartNumber, chartTime, valueText } from "./visualization.js"; + +const ROLLING_SECONDS = 60; +const chartRegistry = new Map(); +let chartSeq = 0; + +export function resetCharts() { + chartRegistry.clear(); + chartSeq = 0; +} + +export function renderChart({ + id, + title, + unit = "", + points = [], + start, + end, + min, + max, + tone = "neutral", + gapSeconds = Infinity, + empty = "No samples in this time range", + note = "", + steps = false, + readingSeconds = 0, + bands = [], +}) { + const chartKey = id || title || `chart-${chartSeq}`; + const uid = `chart-${chartSeq++}`; + const mode = ui.chartModes.get(chartKey) ?? (bands.length ? "running" : "readings"); + + const running = []; + if (mode === "running") { + const raw = bands.flatMap((band) => band.spans).map(([a, b]) => [Math.max(start, a), Math.min(end, b)]).filter(([a, b]) => b > a).toSorted((p, q) => p[0] - q[0]); + for (const [a, b] of raw) { + if (running.length && a <= running.at(-1)[1]) running.at(-1)[1] = Math.max(running.at(-1)[1], b); + else running.push([a, b]); + } + } + + let shown = points; + if (mode === "running") { + shown = [[start, 0], ...running.flatMap(([a, b]) => [[a, 0], [a, 100], [b, 100], [b, 0]]), [end, 0]]; + } else if (mode === "rolling") { + const sorted = points.filter(([at, value]) => Number.isFinite(at) && Number.isFinite(value)).toSorted((a, b) => a[0] - b[0]); + let first = 0, sum = 0; + shown = sorted.map(([at, value], i) => { + sum += value; + while (sorted[first][0] <= at - ROLLING_SECONDS) sum -= sorted[first++][1]; + return [at, sum / (i - first + 1)]; + }); + } + + const samples = shown.filter(([at]) => Number.isFinite(at) && at >= start && at <= end).sort((a, b) => a[0] - b[0]); + const data = samples.filter(([, value]) => Number.isFinite(value)); + const values = data.map(([, value]) => value); + + const modeButtons = readingSeconds + ? `${bands.length ? `` : ""}` + : ""; + const latest = data.length && mode !== "running" + ? `Latest ${escape(valueText(data.at(-1)[1], unit))}` + : ""; + const head = `

${escape(title)}

${modeButtons}${latest}
`; + + if (!data.length) { + return `
${head}

${escape(empty)}

`; + } + + let low = Number.isFinite(min) ? min : Math.min(...values); + let high = Number.isFinite(max) ? max : Math.max(...values); + if (high <= low) { + const pad = Math.max(Math.abs(low) * 0.1, 0.1); + if (!Number.isFinite(min)) low -= pad; + high += pad; + } + let ticks = [high, (high + low) / 2, low]; + if (!Number.isFinite(min) && !Number.isFinite(max)) { + const raw = (high - low) / 3, magnitude = 10 ** Math.floor(Math.log10(raw)); + const step = [1, 2, 5, 10].find((n) => n >= raw / magnitude) * magnitude; + low = Math.floor(low / step) * step; + high = Math.ceil(high / step) * step; + ticks = Array.from({ length: Math.round((high - low) / step) + 1 }, (_, i) => Number((high - i * step).toPrecision(12))); + } + + const segments = []; + let current = null; + for (const sample of samples) { + if (!Number.isFinite(sample[1])) { current = null; continue; } + if (!current || (mode !== "running" && sample[0] - current.at(-1)[0] > gapSeconds)) segments.push((current = [])); + current.push(sample); + } + const spans = segments.map((segment) => [segment[0][0], segment.at(-1)[0]]); + const xAt = (at) => (Math.max(start, Math.min(end, at)) - start) / (end - start || 1) * 1000; + const x = (at) => ((at - start) / (end - start || 1) * 1000).toFixed(1); + const y = (value) => (1000 - (Math.max(low, Math.min(high, value)) - low) / (high - low) * 1000).toFixed(1); + + const paths = segments.map((segment) => { + const line = segment.map(([at, value], i) => !i ? `M${x(at)},${y(value)}` : readingSeconds && mode === "readings" ? `H${x(at)} V${y(value)}` : `L${x(at)},${y(value)}`).join(" "); + return { line: segment.length === 1 ? `${line} h0.01` : line, area: segment.length > 1 ? `${line} L${x(segment.at(-1)[0])},1000 L${x(segment[0][0])},1000 Z` : null }; + }); + + const gridSvg = ticks.map((tick) => ``).join(""); + const areasSvg = paths.filter((p) => p.area).map((p) => ``).join(""); + const bandsSvg = bands.map((band, i) => { + const rects = band.spans.map(([a, b]) => ``).join(""); + const fills = paths.filter((p) => p.area).map((p) => ``).join(""); + return `${rects}${fills}`; + }).join(""); + const linesSvg = paths.map((p) => ``).join(""); + const crosshairSvg = steps ? "" : ``; + + const tickCount = end === start ? 0 : 4; + const label = (at) => steps ? chartNumber(Math.round(at)) : axisTime(at, { start, now: end }); + const xLabels = Array.from({ length: tickCount + 1 }, (_, i) => `${escape(label(start + (end - start) * i / (tickCount || 1)))}`).join(""); + const yLabels = ticks.map((tick) => `${escape(chartNumber(tick))}`).join(""); + + const summary = mode === "running" + ? `Something ran ${escape(valueText(100 * running.reduce((sum, [a, b]) => sum + b - a, 0) / (end - start || 1), "%"))} of the range` + : `Average ${escape(valueText(values.reduce((sum, v) => sum + v, 0) / values.length, unit))}Peak ${escape(valueText(Math.max(...values), unit))}${readingSeconds ? `${escape(mode === "rolling" ? `Mean of the last minute's ${readingSeconds} s readings` : `One reading per ${readingSeconds} s`)}` : ""}`; + const bandKeys = bands.map((band) => `${escape(band.label)}`).join(""); + const noteHtml = note ? `${escape(note)}` : ""; + + chartRegistry.set(uid, { data, spans, start, end, low, high, unit, steps, index: -1 }); + + return `
${head} +
+
${yLabels}
+ ${gridSvg}${areasSvg}${bandsSvg}${linesSvg}${crosshairSvg} +
${xLabels}
+ +
+
${summary}${bandKeys}${noteHtml}
+
`; +} + +function indexAt(data, at) { + let left = 0, right = data.length - 1; + while (left < right) { + const middle = Math.floor((left + right) / 2); + if (data[middle][0] < at) left = middle + 1; + else right = middle; + } + return left > 0 && Math.abs(data[left - 1][0] - at) < Math.abs(data[left][0] - at) ? left - 1 : left; +} + +function showSample(plot, entry, index) { + const hover = plot.querySelector(".chart-hover"), svg = plot.querySelector("svg"); + if (!entry.data.length || !hover || !svg) return; + entry.index = Math.max(0, Math.min(entry.data.length - 1, index)); + const [at, value] = entry.data[entry.index], box = svg.getBoundingClientRect(); + const left = (at - entry.start) / (entry.end - entry.start || 1) * box.width; + hover.hidden = false; + hover.style.left = `${box.left - plot.getBoundingClientRect().left + left}px`; + hover.querySelector(".chart-point").style.top = `${Math.max(0, Math.min(100, (entry.high - value) / (entry.high - entry.low) * 100))}%`; + const tip = hover.querySelector(".chart-tip"); + tip.textContent = `${entry.steps ? `Step ${at}` : chartTime(at, true)} · ${valueText(value, entry.unit)}`; + tip.style.left = `${Math.max(-left, Math.min(8, box.width - left - (tip.offsetWidth || 160)))}px`; +} + +export function hoverChart(plot, event) { + const entry = chartRegistry.get(plot.dataset.chartId); + const svg = plot.querySelector("svg"), hover = plot.querySelector(".chart-hover"); + if (!entry || !svg || !hover) return; + const box = svg.getBoundingClientRect(); + const at = entry.start + (event.clientX - box.left) / (box.width || 1) * (entry.end - entry.start); + const outside = event.clientX < box.left || event.clientX > box.right; + if (!entry.steps) setCrosshair(ui.navigating || event.buttons || outside ? null : at, { start: entry.start, now: entry.end }); + if (!entry.data.length || ui.navigating || event.buttons) { hover.hidden = true; return; } + const idx = indexAt(entry.data, at); + const near = Math.abs(entry.data[idx][0] - at) / (entry.end - entry.start || 1) * box.width < 6; + if (outside || (!near && !entry.spans.some(([a, b]) => at >= a && at <= b))) hover.hidden = true; + else showSample(plot, entry, idx); +} + +export function inspectChart(plot, key) { + if (!["ArrowLeft", "ArrowRight", "Home", "End", "Escape"].includes(key)) return false; + const entry = chartRegistry.get(plot.dataset.chartId); + if (!entry) return false; + if (key === "Escape") { + const hover = plot.querySelector(".chart-hover"); + if (hover) hover.hidden = true; + plot.blur(); + return true; + } + const idx = key === "Home" ? 0 : key === "End" ? entry.data.length - 1 : (entry.index < 0 ? entry.data.length - 1 : entry.index) + (key === "ArrowLeft" ? -1 : 1); + showSample(plot, entry, idx); + return true; +} + +export function timeControlHtml(range, observed) { + const duration = range.now - range.start; + const custom = [[Math.floor(duration / 3600), "h"], [Math.floor(duration / 60) % 60, "m"], [duration % 60, "s"]].filter(([n]) => n).map(([n, unit]) => `${n}${unit}`).join(" "); + const options = WINDOWS.some(([n]) => n === duration) ? WINDOWS : [[duration, `${custom} (custom)`], ...WINDOWS]; + const local = (at) => new Date(at * 1000).toISOString().slice(0, duration <= 120 ? 19 : 16); + const timeLabel = range.live + ? `Last ${WINDOWS.find(([n]) => n === duration)?.[1] || custom}` + : `${local(range.start).slice(5, 10)} · ${chartTime(range.start, duration <= 120)} – ${local(range.start).slice(0, 10) === local(range.now).slice(0, 10) ? "" : `${local(range.now).slice(5, 10)} · `}${chartTime(range.now, duration <= 120)}`; + return ` +
${escape(timeLabel)} +
+ + ${range.live ? 'Following current time' : ''} + W/S zoom · A/D pan +
+
+ `; +} diff --git a/src/server/observability/static/dom.js b/src/server/observability/static/dom.js new file mode 100644 index 000000000..7369c88bf --- /dev/null +++ b/src/server/observability/static/dom.js @@ -0,0 +1,73 @@ +// Keyed DOM morphing so periodic refreshes keep focus, open popovers, scroll position, and hover state. + +const keyOf = (node) => node.nodeType === 1 && (node.id || node.getAttribute("data-key")); + +export function morph(container, html) { + const template = document.createElement("template"); + template.innerHTML = html; + morphChildren(container, template.content); +} + +function morphChildren(from, to) { + const keyed = new Map(Array.from(from.childNodes).filter(keyOf).map((node) => [keyOf(node), node])); + let cursor = from.firstChild; + for (const next of Array.from(to.childNodes)) { + const key = keyOf(next); + let node = key ? keyed.get(key) : cursor && !keyOf(cursor) ? cursor : null; + if (!node) from.insertBefore((node = next), cursor); + else { + if (node !== cursor) from.insertBefore(node, cursor); + node = morphNode(node, next); + } + cursor = node.nextSibling; + } + while (cursor) { + const next = cursor.nextSibling; + cursor.remove(); + cursor = next; + } +} + +function morphNode(node, next) { + if (node.isEqualNode(next)) return node; + if (node.nodeType !== next.nodeType || (node.nodeType === 1 && (node.tagName !== next.tagName || node.namespaceURI !== next.namespaceURI))) { + node.replaceWith(next); + return next; + } + if (node.nodeType !== 1) { + if (node.data !== next.data) node.data = next.data; + return node; + } + if (node.classList.contains("chart-hover") || node.classList.contains("activity-tooltip")) return node; + if (node.classList.contains("chart-plot") && ["chartId", "start", "end", "min", "max"].some((k) => node.dataset[k] !== next.dataset[k])) { + const hover = node.querySelector(".chart-hover"); + if (hover) hover.hidden = true; + } + const preserve = (name) => (name === "open" && node.tagName === "DETAILS") || (name === "data-crosshair" || name === "style" && node.hasAttribute("data-crosshair-host")); + const nextValue = next.value, nextChecked = next.checked; + for (const { name } of Array.from(node.attributes)) if (!preserve(name) && !next.hasAttribute(name)) node.removeAttribute(name); + for (const { name, value } of Array.from(next.attributes)) if (!preserve(name) && node.getAttribute(name) !== value) node.setAttribute(name, value); + morphChildren(node, next); + if ((node.tagName === "SELECT" || (node.tagName === "INPUT" && node.type !== "checkbox" && node !== document.activeElement)) && node.value !== nextValue) { + node.value = nextValue; + } + if (node.tagName === "INPUT" && node.type === "checkbox" && node.checked !== nextChecked) node.checked = nextChecked; + return node; +} + +export async function copyLink(url, success = "Link copied") { + try { + await navigator.clipboard.writeText(url); + return success; + } catch { + const focused = document.activeElement, input = Object.assign(document.createElement("textarea"), { value: url }); + input.style.cssText = "position:fixed;opacity:0"; + document.body.append(input); + input.select(); + let copied = false; + try { copied = document.execCommand("copy"); } catch {} + input.remove(); + focused?.focus({ preventScroll: true }); + return copied ? success : "Copy failed"; + } +} diff --git a/src/server/observability/static/format.js b/src/server/observability/static/format.js new file mode 100644 index 000000000..a6e9c8750 --- /dev/null +++ b/src/server/observability/static/format.js @@ -0,0 +1,30 @@ +export const escape = (value) => String(value ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c]); +export const duration = (total) => total < 90 ? `${Math.round(total)}s` : total < 5400 ? `${Math.round(total / 60)}m` : total < 172800 ? `${(total / 3600).toFixed(1)}h` : `${(total / 86400).toFixed(1)}d`; +const seconds = (value) => value === null || value === undefined || value === "" ? NaN : typeof value === "number" ? value : Date.parse(value) / 1000; +export const isFinished = (run) => run?.lifecycle === "ended"; +export const finishedAt = (run) => [seconds(run?.completed_at), run?.last_completed_at].find(Number.isFinite) ?? NaN; +export const runLabel = (run) => run.client_status !== "active" ? run.client_status : { ended: "ended", failed: "failed", waiting: "queued", placed: "running" }[run.lifecycle] || "active"; +export function elapsedTime(run, observedAt) { + const stop = isFinished(run) ? finishedAt(run) : NaN; + const start = seconds(run.created_at), end = Number.isFinite(stop) ? stop : seconds(observedAt); + return Number.isFinite(start) && Number.isFinite(end) ? duration(Math.max(0, end - start)) : "—"; +} +export function shortNodeName(name, nodes = []) { + const full = String(name || "—"), parts = full.split("-"); + const others = nodes.map((n) => typeof n === "string" ? n : n.name).filter((n) => n && n !== full); + for (let i = parts.length - 1; i >= 0; i--) { + const suffix = parts.slice(i).join("-"); + if (suffix && !others.some((other) => other === suffix || other.endsWith(`-${suffix}`))) return suffix; + } + return full; +} +export const KINDS = { lora: "LoRA", full: "FFT", fft: "FFT" }; +export const runDescription = (run, title = run.name) => [run.display_name, run.recipe_name].filter((v, i, values) => v && v !== title && values.indexOf(v) === i).join(" · "); +export const escapeRegExp = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +export const family = (id) => `hue-${Array.from(id || "").reduce((sum, c) => sum + c.charCodeAt(0), 0) % 8}`; +export function statusBadge(value) { + const words = value === "waiting_for_placement" ? "queued" : String(value || "Unknown"); + const label = words[0].toUpperCase() + words.slice(1); + const tone = { running: "running", ready: "running", live: "running", stalled: "failed", failed: "failed", completed: "completed", ended: "completed", stopped: "neutral" }[label.toLowerCase()] || "pending"; + return `${escape(label)}`; +} diff --git a/src/server/observability/static/index.html b/src/server/observability/static/index.html new file mode 100644 index 000000000..e5b5b63bf --- /dev/null +++ b/src/server/observability/static/index.html @@ -0,0 +1,12 @@ + + + + + + + OpenRL · Cluster + + + +
+ diff --git a/src/server/observability/static/node-time.js b/src/server/observability/static/node-time.js new file mode 100644 index 000000000..91b329384 --- /dev/null +++ b/src/server/observability/static/node-time.js @@ -0,0 +1,162 @@ +// Time-axis drag, Ctrl/⌘+wheel zoom, two-finger pinch, and W/S A/D navigation. +import { MAX_WINDOW, now, rangeOf, setCrosshair, ui, view } from "./state.js"; + +const clamp = (value, low, high) => Math.max(low, Math.min(high, value)); +const SURFACES = ".capacity-track, .node-axis, .activity-track, .activity-axis, #placement-detail .chart-plot svg, #placement-detail .chart-empty"; + +export function attachNodeTime(root) { + let gesture = null, drag = null, pinch = null, settle, ignoreClickUntil = 0; + let pointerAt = null; + const getNow = () => now(ui.inventory.data); + const getRange = () => rangeOf(ui.inventory.data, view); + const fraction = (x, box) => clamp((x - box.left) / (box.width || 1), 0, 1); + + function begin(kind) { + if (gesture) return; + gesture = { kind, duration: view.duration, end: view.end }; + ui.navigating = true; + root.classList.add("node-navigating"); + } + + function select(duration, end) { + duration = clamp(Math.round(duration), 60, MAX_WINDOW); + view.duration = duration; + view.end = clamp(end, getNow() - MAX_WINDOW + duration, getNow()); + ui.render(); + } + + function finish(cancel = false) { + clearTimeout(settle); + if (drag?.pointerId !== undefined && root.hasPointerCapture(drag.pointerId)) root.releasePointerCapture(drag.pointerId); + drag = pinch = null; + if (!gesture) return; + if (cancel) { + view.duration = gesture.duration; + view.end = gesture.end; + } + if (gesture.kind === "drag") ignoreClickUntil = performance.now() + 400; + gesture = null; + ui.navigating = false; + root.classList.remove("node-navigating"); + ui.render(); + } + + function moveDrag(x, y, touch = false) { + if (!drag) return false; + const dx = x - drag.x, dy = y - drag.y; + if (!gesture) { + if (touch && Math.abs(dy) > 6 && Math.abs(dy) > Math.abs(dx)) { drag = null; return false; } + if (Math.abs(dx) < 6) return false; + begin("drag"); + if (drag.pointerId !== undefined) root.setPointerCapture(drag.pointerId); + } + select(drag.range.now - drag.range.start, drag.range.now - dx / (drag.box.width || 1) * (drag.range.now - drag.range.start)); + return true; + } + + const activePage = () => view.page === "nodes" || view.page === "run"; + + root.addEventListener("pointermove", (event) => { + if (!activePage()) { pointerAt = null; return; } + const surface = event.target.closest(".capacity-track, .activity-track, .chart-plot svg"); + if (!surface || ui.navigating || event.buttons) { pointerAt = null; return; } + const box = surface.getBoundingClientRect(), range = getRange(); + pointerAt = range.start + fraction(event.clientX, box) * (range.now - range.start); + }); + + root.addEventListener("wheel", (event) => { + if (!activePage()) return; + const track = event.target.closest(SURFACES); + if (!track || !(event.ctrlKey || event.metaKey) || (gesture && gesture.kind !== "wheel")) return; + event.preventDefault(); + begin("wheel"); + const range = getRange(), at = fraction(event.clientX, track.getBoundingClientRect()); + const delta = event.deltaY * (event.deltaMode === 1 ? 16 : event.deltaMode === 2 ? track.clientHeight : 1); + const duration = clamp(Math.round((range.now - range.start) * Math.exp(clamp(delta * 0.002, -1, 1))), 60, MAX_WINDOW); + select(duration, range.start + at * (range.now - range.start) + (1 - at) * duration); + clearTimeout(settle); + settle = setTimeout(() => finish(), 200); + }, { passive: false }); + + root.addEventListener("pointerdown", (event) => { + if (!activePage()) return; + const track = event.target.closest(SURFACES); + if (!track || event.pointerType === "touch" || event.button !== 0 || event.target.closest("[data-placement], a")) return; + finish(); + drag = { x: event.clientX, y: event.clientY, box: track.getBoundingClientRect(), range: getRange(), pointerId: event.pointerId }; + }); + window.addEventListener("pointermove", (event) => { if (drag?.pointerId === event.pointerId && moveDrag(event.clientX, event.clientY)) event.preventDefault(); }); + window.addEventListener("pointerup", (event) => { if (drag?.pointerId === event.pointerId) finish(); }); + window.addEventListener("pointercancel", (event) => { if (drag?.pointerId === event.pointerId) finish(true); }); + + root.addEventListener("touchstart", (event) => { + if (!activePage()) return; + const track = event.target.closest(SURFACES); + if (!track) return; + if (event.touches.length === 2) { + event.preventDefault(); + if (gesture?.kind === "wheel") finish(); + begin("pinch"); + gesture.kind = "pinch"; + drag = null; + const [a, b] = event.touches; + pinch = { box: track.getBoundingClientRect(), range: getRange(), x: (a.clientX + b.clientX) / 2, distance: Math.hypot(a.clientX - b.clientX, a.clientY - b.clientY) || 1 }; + } else if (event.touches.length === 1 && !event.target.closest("[data-placement], a")) { + const touch = event.touches[0]; + drag = { x: touch.clientX, y: touch.clientY, box: track.getBoundingClientRect(), range: getRange() }; + } + }, { passive: false }); + + root.addEventListener("touchmove", (event) => { + if (pinch && event.touches.length === 2) { + event.preventDefault(); + const [a, b] = event.touches, x = (a.clientX + b.clientX) / 2; + const duration = clamp(Math.round((pinch.range.now - pinch.range.start) * pinch.distance / (Math.hypot(a.clientX - b.clientX, a.clientY - b.clientY) || 1)), 60, MAX_WINDOW); + const anchor = pinch.range.start + fraction(pinch.x, pinch.box) * (pinch.range.now - pinch.range.start); + select(duration, anchor + (1 - fraction(x, pinch.box)) * duration); + } else if (drag && event.touches.length === 1 && moveDrag(event.touches[0].clientX, event.touches[0].clientY, true)) event.preventDefault(); + }, { passive: false }); + root.addEventListener("touchend", () => { if (drag || pinch) finish(); }); + root.addEventListener("touchcancel", () => { if (drag || pinch) finish(true); }); + + root.addEventListener("click", (event) => { + if (event.detail && performance.now() < ignoreClickUntil) { + ignoreClickUntil = 0; + event.preventDefault(); + event.stopImmediatePropagation(); + } + }, { capture: true }); + + window.addEventListener("keydown", (event) => { + if (event.key === "Escape" && (gesture || drag || pinch)) { + event.preventDefault(); + event.stopImmediatePropagation(); + finish(true); + } + }, { capture: true }); + + window.addEventListener("keydown", (event) => { + if (!activePage()) return; + const key = event.key.toLowerCase(); + if (!["w", "a", "s", "d"].includes(key) || event.ctrlKey || event.metaKey || event.altKey || event.target.closest("input, select, textarea") || (gesture && gesture.kind !== "keys")) return; + event.preventDefault(); + begin("keys"); + const range = getRange(), span = range.now - range.start; + if (key === "a" || key === "d") { + select(span, range.now + (key === "d" ? 0.2 : -0.2) * span); + if (pointerAt !== null) { + pointerAt += view.end - range.now; + setCrosshair(pointerAt, getRange()); + } + } else { + const duration = clamp(Math.round(span * (key === "w" ? 0.8 : 1.25)), 60, MAX_WINDOW); + const at = pointerAt === null ? 0.5 : clamp((pointerAt - range.start) / span, 0, 1); + select(duration, range.start + at * span + (1 - at) * duration); + } + clearTimeout(settle); + settle = setTimeout(() => finish(), 300); + }); + + window.addEventListener("blur", () => finish(true)); + window.addEventListener("hashchange", () => finish()); +} diff --git a/src/server/observability/static/state.js b/src/server/observability/static/state.js new file mode 100644 index 000000000..5f3ef221e --- /dev/null +++ b/src/server/observability/static/state.js @@ -0,0 +1,231 @@ +// URL state, fetch client, and render-scoped query cache. + +export const MAX_WINDOW = 7 * 86400; +export const WINDOWS = [[60, "1 minute"], [600, "10 minutes"], [1800, "30 minutes"], [3600, "1 hour"], [10800, "3 hours"], [21600, "6 hours"], [86400, "24 hours"], [259200, "3 days"], [MAX_WINDOW, "7 days"]]; + +const queryString = (params) => { + const query = new URLSearchParams(Object.entries(params).filter(([, v]) => v !== null && v !== undefined && v !== "")).toString(); + return query ? `?${query}` : ""; +}; +export const viewLink = (path, params = {}) => `#${path}${queryString(params)}`; +export const inspectUrl = (resource, params = {}) => `/api/v1/inspect/${resource}${queryString(params)}`; +export const nodeLink = (placement, range) => viewLink("nodes", { placement, duration: range.now - range.start, end: range.now }); +export const now = (inventory) => Date.parse(inventory?.observed_at) / 1000 || Date.now() / 1000; + +export const saved = { overview: "#overview", nodes: "#nodes" }; +export const ui = { + inventory: { data: null, error: null, status: null, pending: false, stale: false, failures: 0, etag: null }, + navigating: false, + activityMode: "lanes", + openGpus: new Set(), + chartModes: new Map(), + runOpened: new Set(), + scrollSelection: true, + render: () => {}, +}; + +const number = (params, key, fallback) => params.get(key)?.trim() && Number.isFinite(Number(params.get(key))) ? Number(params.get(key)) : fallback; + +export function parseView(hash = location.hash) { + const [path, search = ""] = hash.slice(1).split("?"); + const [page, id = "", tab = "activity"] = path.split("/").map((part) => { try { return decodeURIComponent(part); } catch { return part; } }); + const params = new URLSearchParams(search), duration = Math.max(60, Math.min(MAX_WINDOW, Math.round(number(params, "duration", 1800)))); + const end = number(params, "end", null), event = number(params, "event", null), back = params.get("back"); + return { + page: ["nodes", "run", "health"].includes(page) ? page : "overview", + id, + tab, + explicitWindow: params.has("duration") || params.has("end"), + duration, + end: end !== null && end >= duration && end <= 8.64e12 ? end : null, + event: event !== null && (end === null || (event >= end - duration && event <= end)) ? event : null, + back: back?.startsWith("#nodes?") ? back : null, + node: params.get("node"), + placement: params.get("placement"), + gpu: params.get("gpu") || "all", + q: params.get("q") || "", + pod: params.get("pod") || "", + status: ["active", "queued", "finished"].includes(params.get("status")) ? params.get("status") : "all", + }; +} + +export const view = parseView(); + +export const rangeOf = (inventory, selection = view) => { + const observed = now(inventory), duration = Math.max(60, Math.min(MAX_WINDOW, Math.round(selection.duration || 1800))); + const end = Math.max(observed - MAX_WINDOW + duration, Math.min(observed, selection.end ?? observed)); + return { start: end - duration, now: end, live: selection.end === null }; +}; + +export function withoutSelection(hash) { + const [path, query = ""] = hash.split("?"); + const params = new URLSearchParams(query); + for (const name of ["node", "placement", "gpu"]) params.delete(name); + return params.size ? `${path}?${params}` : path; +} + +export function currentView(freeze = false, tab = view.tab) { + const { page, id, duration, end, back, event, q, pod, node, placement, gpu, status } = view; + if (page === "overview") return viewLink(page, { q, status: status === "all" ? null : status }); + if (page === "run") return viewLink(`run/${encodeURIComponent(id)}/${tab || "activity"}`, { duration, end: freeze ? end ?? now(ui.inventory.data) : end, back, event, q, pod }); + if (page === "nodes") return viewLink(page, { duration, end: freeze ? end ?? now(ui.inventory.data) : end, node, placement, gpu: node && gpu !== "all" ? gpu : null }); + return viewLink(page + (id ? `/${encodeURIComponent(id)}` : "")); +} + +export function syncHash() { + const hash = currentView(); + if (location.hash !== hash) history.replaceState(history.state, "", hash); + if (view.page === "overview") saved.overview = hash; + if (view.page === "nodes" || (view.page === "run" && view.back)) saved.nodes = withoutSelection(view.page === "nodes" ? hash : view.back); +} + +export async function get(url, signal, previous) { + const timeout = AbortSignal.timeout(15000); + let response; + try { + response = await fetch(url, { cache: "no-store", headers: previous?.etag ? { "If-None-Match": previous.etag } : undefined, signal: signal ? AbortSignal.any([signal, timeout]) : timeout }); + } catch (error) { + if (signal?.aborted) throw error; + throw new Error(timeout.aborted ? "Dashboard request timed out. Try again." : "Cannot reach the dashboard. Check the connection and try again."); + } + if (response.status === 304 && previous?.etag) return previous; + if (!response.ok) { + const data = await response.json().catch(() => null); + const detail = Array.isArray(data?.detail) ? data.detail.map((item) => `${(item.loc || []).filter((part) => part !== "body").join(".")}: ${item.msg}`).join("; ") : data?.detail; + const error = new Error([detail, data?.error].find((v) => typeof v === "string") || ({ 401: "Authentication required", 403: "Access denied", 404: "Requested data not found", 429: "Too many requests. Try again shortly." }[response.status] || `Dashboard request failed (${response.status})`)); + error.status = response.status; + throw error; + } + const data = await response.json().catch(() => { throw new Error("The dashboard returned an invalid response. Try again."); }); + return { data, etag: response.headers.get("ETag") }; +} + +// Render-scoped query cache: pages request a URL under a scope key; fetches run in the background and coalesce repaints. +const entries = new Map(); +const remembered = new Map(); +const MAX_ENTRIES = 24; +let generation = 0; +let repaint = 0; + +function scheduleRepaint() { + cancelAnimationFrame(repaint); + const g = generation; + repaint = requestAnimationFrame(() => { if (g === generation) ui.render(); }); +} + +export const beginRender = () => { generation++; }; +export function endRender() { + for (const [scope, entry] of entries) { + if (entry.used === generation) continue; + entry.controller?.abort(); + clearTimeout(entry.timer); + entries.delete(scope); + } +} + +export function retryQuery(scope) { + const entry = entries.get(scope); + if (!entry) return; + entry.controller?.abort(); + clearTimeout(entry.timer); + entry.fetchedAt = 0; + entry.failures = 0; + entry.pending = false; + ui.render(); +} + +export function use(url, { scope = url, poll = 15000, pollWhen = true, remember = false } = {}) { + if (!url) return { data: null, error: null, status: null, pending: false, stale: false }; + let entry = entries.get(scope); + if (!entry) { + const cached = remembered.get(url); + entry = { scope, url: null, data: cached?.data ?? null, etag: cached?.etag ?? null, error: null, status: null, fetchedAt: cached ? Date.now() : 0, pending: false, stale: false, failures: 0, controller: null, timer: null, used: generation }; + entries.set(scope, entry); + } + entry.used = generation; + const urlChanged = entry.url !== url; + if (urlChanged && remembered.has(url)) { + const cached = remembered.get(url); + entry.url = url; + entry.data = cached.data; + entry.etag = cached.etag; + entry.error = entry.status = null; + entry.stale = false; + entry.fetchedAt = Date.now(); + } + const due = urlChanged || (poll > 0 && pollWhen && Date.now() - entry.fetchedAt >= poll); + if (!ui.navigating && !entry.pending && due) { + entry.controller?.abort(); + clearTimeout(entry.timer); + entry.url = url; + entry.pending = true; + const controller = entry.controller = new AbortController(); + get(url, controller.signal, entry.etag ? entry : null) + .then(({ data, etag }) => { + if (controller.signal.aborted) return; + entry.failures = 0; + entry.error = data.error || null; + entry.status = null; + entry.stale = false; + if (!entry.error || !entry.data) entry.data = data; + entry.etag = etag; + if (!entry.error && remember) { + remembered.set(url, { data: entry.data, etag }); + if (remembered.size > MAX_ENTRIES) remembered.delete(remembered.keys().next().value); + } + }) + .catch((error) => { + if (controller.signal.aborted) return; + entry.failures += 1; + const transient = (!error.status || error.status >= 500 || error.status === 429) && entry.failures < 4; + if (transient) { + entry.stale = Boolean(entry.data); + entry.timer = setTimeout(() => { if (entries.get(scope) === entry) { entry.fetchedAt = 0; ui.render(); } }, 2000 * 2 ** (entry.failures - 1)); + } else { + entry.error = error.message; + entry.status = error.status || null; + entry.stale = false; + } + }) + .finally(() => { + if (!controller.signal.aborted) { + entry.pending = false; + entry.controller = null; + entry.fetchedAt = Date.now(); + if (entries.get(scope) === entry && entry.used === generation) { + if (poll > 0 && pollWhen && !entry.timer) { + entry.timer = setTimeout(() => { if (!document.hidden && entries.get(scope) === entry) ui.render(); }, poll); + } + scheduleRepaint(); + } + } + }); + } + return entry; +} + +export const timeAt = (event, range) => { + const box = event.currentTarget.getBoundingClientRect(); + return range.start + Math.max(0, Math.min(1, (event.clientX - box.left) / (box.width || 1))) * (range.now - range.start); +}; + +// Update shared crosshair directly on DOM hosts (no full re-morph on pointermove). +export function setCrosshair(at, range) { + const pct = at === null || !range || at < range.start || at > range.now ? null : 100 * (at - range.start) / (range.now - range.start || 1); + for (const host of document.querySelectorAll("[data-crosshair-host]")) { + if (pct === null) host.removeAttribute("data-crosshair"); + else { + host.setAttribute("data-crosshair", ""); + host.style.setProperty("--crosshair", `${pct}%`); + } + } + for (const line of document.querySelectorAll("line.chart-crosshair")) { + if (pct === null) line.setAttribute("visibility", "hidden"); + else { + line.removeAttribute("visibility"); + const x = (pct * 10).toFixed(1); + line.setAttribute("x1", x); + line.setAttribute("x2", x); + } + } +} diff --git a/src/server/observability/static/style.css b/src/server/observability/static/style.css new file mode 100644 index 000000000..7e1be2830 --- /dev/null +++ b/src/server/observability/static/style.css @@ -0,0 +1,351 @@ +:root { color-scheme: light dark; /* A scrollbar appearing as content loads would otherwise narrow and reflow the page. */ scrollbar-gutter: stable; } +body { margin: 0; background: light-dark(#ffffff, #131519); } +#app { --ink: light-dark(#202328, #e6e8ed); --dim: light-dark(#626874, #a0a6b1); --ground: light-dark(#ffffff, #131519); --paper: light-dark(#f7f8fa, #191c21); --line: light-dark(#e3e6eb, #2a2e35); --hover: light-dark(#eff1f4, #20242b); --selected: light-dark(#e8ebf0, #282d35); --control-border: light-dark(#c9ced7, #424954); --accent: light-dark(#416bc3, #83a9ff); --hue-0: var(--accent); --hue-1: light-dark(#357e70, #76b3a2); --hue-2: light-dark(#7a60bb, #ad98dc); --hue-3: light-dark(#a87434, #c3a072); --hue-4: light-dark(#a8496b, #cd8ca5); --hue-5: light-dark(#2f8a9a, #7cb3bd); --hue-6: light-dark(#858533, #b5b579); --hue-7: light-dark(#b85538, #e39a7f); --status-success: light-dark(#27725c, #83baa5); --status-warning: light-dark(#895e1e, #ccb079); --status-error: light-dark(#ab4249, #da949a); --status-complete: light-dark(#496a97, #9bb0ce); --radius: 5px; --gutter: 22px; --node-label-width: 200px; --gpu-index-width: 38px; --gpu-index-gap: 6px; --sans: Inter, ui-sans-serif, system-ui, sans-serif; --mono: "SFMono-Regular", Consolas, monospace; color: var(--ink); background: var(--ground); width: 100%; min-height: 100dvh; font: 13px/1.5 var(--sans); -webkit-font-smoothing: antialiased; } +#app * { box-sizing: border-box; } +#app :is(h1, h2, p, figure) { margin: 0; } +#app a { color: inherit; text-decoration: none; } +#app a:hover { text-decoration: underline; text-underline-offset: 3px; } +#app button { cursor: pointer; font: inherit; color: inherit; } +#app :where(input, select, .chip, .time-shift) { min-height: 32px; max-width: 100%; padding: 5px 9px; border: 1px solid var(--line); border-radius: var(--radius); background-color: var(--paper); color: var(--ink); font: inherit; line-height: 20px; box-shadow: none; } +#app select { appearance: none; padding-right: 30px; background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='8' viewBox='0 0 12 8'%3E%3Cpath d='m1 1.5 5 5 5-5' fill='none' stroke='%23878d98' stroke-width='1.4' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E"); background-repeat: no-repeat; background-position: right 10px center; } +#app input::placeholder { color: var(--dim); opacity: 1; } +#app :is(input, select):hover, #app .chip:hover:not(:disabled) { border-color: var(--control-border); } +#app .chip:hover:not(:disabled) { background-color: var(--hover); } +#app .chip[aria-pressed="true"] { background: var(--selected); border-color: var(--control-border); } +#app button:disabled { cursor: default; opacity: 0.5; } +#app :is(a, button, input, select):focus-visible { outline: 1px solid var(--accent); outline-offset: 3px; } +#app :is(input, select):focus { border-color: var(--control-border); box-shadow: none; } +#app :is(input, select):focus:not(:focus-visible) { outline: none; } +#app .appbar { display: flex; flex-wrap: wrap; align-items: center; gap: 12px 28px; padding: 14px var(--gutter); border-bottom: 1px solid var(--line); } +#app nav { display: flex; flex-wrap: wrap; gap: 20px; } +#app nav a { display: inline-flex; align-items: center; min-height: 32px; color: var(--dim); } +#app nav a:hover { color: var(--ink); text-decoration: none; } +#app nav [aria-current="page"] { color: var(--ink); text-decoration: underline; text-decoration-thickness: 2px; text-underline-offset: 8px; } +#app .brand { font-size: 16px; font-weight: 500; } +#app #connection { margin-left: auto; color: var(--dim); font-size: 12px; } +#app .pad { padding: 22px var(--gutter); } +#app .heading { margin-bottom: 16px; font-size: 22px; font-weight: 500; line-height: 1.3; overflow-wrap: anywhere; } +#app .mono { font-family: var(--mono); } +#app .muted { color: var(--dim); } +#app .micro { font-size: 11px; } +#app .empty-state { padding: 16px 0; color: var(--dim); } +/* Jobs, processes and health share the same table rhythm. */ +#app :is(.job-list-head, .job-list-row) { display: grid; grid-template-columns: minmax(0, 1fr) 112px 100px 112px 100px; align-items: center; gap: 16px; padding: 12px 0; } +#app .job-list-head { color: var(--dim); font-size: 12px; } +#app .job-list-row { border-top: 1px solid var(--line); } +#app .job-list-row > * { min-width: 0; overflow-wrap: anywhere; } +#app .job-list-row:hover { text-decoration: none; } +#app .job-list-row:hover .job-identity > span:first-child { text-decoration: underline; text-underline-offset: 3px; } +#app .job-identity { display: grid; gap: 3px; min-width: 0; overflow-wrap: anywhere; } +#app .job-identity .micro { font-size: 12px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#app :is(.job-list-head, .job-list-row) > :nth-child(n + 4) { text-align: right; font-variant-numeric: tabular-nums; } +/* Overview: search and status chips above the run list. */ +#app .overview-toolbar { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 16px; margin-bottom: 14px; } +#app #run-search { flex: 1 1 280px; max-width: 420px; } +#app .run-filters { display: flex; flex-wrap: wrap; gap: 6px; } +#app .metric-filters { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin-bottom: 12px; } #app .metric-filters input { width: 28ch; } +#app .run-filter-count { margin-left: 4px; color: var(--dim); font-variant-numeric: tabular-nums; } +#app :is(.job-list-head, .job-list-row).run-row { position: relative; grid-template-columns: minmax(0, 1fr) 120px 72px 88px 160px; gap: 24px; } +/* The run link covers its row. */ +#app .run-row .job-identity a::after { content: ""; position: absolute; inset: 0; } +#app .run-row:hover .job-identity a { text-decoration: underline; text-underline-offset: 3px; } +#app :is(.job-list-head, .job-list-row).run-row > :is(:nth-child(3), :nth-child(4)) { text-align: right; font-variant-numeric: tabular-nums; } +#app .run-row > .run-session, #app .job-list-head.run-row > :nth-child(5) { text-align: left; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +/* Sessions group the run rows; a session's runs are indented under its header. */ +#app .state { display: inline-flex; align-items: center; gap: 7px; white-space: nowrap; } +#app :is(.state-dot, .health-dot) { width: 6px; height: 6px; border-radius: 50%; background: currentColor; flex: none; } +#app .state-running { color: var(--status-success); } +#app .state-pending { color: var(--status-warning); } +#app .state-failed { color: var(--status-error); } +#app .state-completed { color: var(--status-complete); } +#app .state-neutral { color: var(--dim); } +#app :is(.run-table, .scheduler-table) { width: 100%; border-collapse: collapse; } +#app :is(.run-table, .scheduler-table) :is(th, td) { padding: 12px 12px 12px 0; border-bottom: 1px solid var(--line); text-align: left; vertical-align: top; overflow-wrap: anywhere; } +#app :is(.run-table, .scheduler-table) :is(th, td):last-child { padding-right: 0; } +#app :is(.run-table, .scheduler-table) th { color: var(--dim); font-weight: 500; } +#app .run-table { table-layout: fixed; min-width: 720px; } +#app .run-table th:first-child { width: 38%; } +#app .run-table :is(th, td):last-child { text-align: right; } +#app .health-status { --health-tone: var(--status-warning); display: inline-flex; align-items: center; gap: 8px; white-space: nowrap; padding: 4px 8px; border-radius: var(--radius); color: var(--health-tone); background: color-mix(in srgb, var(--health-tone) 9%, var(--ground)); } +#app .health-error { --health-tone: var(--status-error); } +#app .health-success { --health-tone: var(--status-success); } +#app .health-message { display: flex; align-items: baseline; flex-wrap: wrap; gap: 8px 12px; margin: 12px 0; overflow-wrap: anywhere; } +/* Nodes: the header, GPU lanes and duty values use one column grid. */ +#app .nodes-heading { display: flex; justify-content: space-between; align-items: center; flex-wrap: wrap; gap: 12px 16px; margin-bottom: 18px; } +#app :is(.nodes-heading, .run-heading) .heading { margin: 0; } +#app .time-control { position: relative; display: grid; grid-template-columns: 32px minmax(0, 1fr) 32px 80px; width: 424px; max-width: 100%; align-items: center; gap: 8px; color: var(--dim); font-size: 12px; } +#app .time-picker { min-width: 0; color: var(--ink); } +#app .time-summary { display: flex; align-items: center; justify-content: space-between; gap: 8px; height: 32px; padding: 5px 10px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--paper); list-style: none; cursor: pointer; line-height: 20px; white-space: nowrap; } +#app .time-summary::-webkit-details-marker { display: none; } +#app .time-summary > span { overflow: hidden; text-overflow: ellipsis; } +#app .time-chevron { display: block; flex: none; width: 16px; height: 16px; fill: none; stroke: var(--dim); stroke-width: 1.5; stroke-linecap: round; stroke-linejoin: round; } +#app .time-summary:hover, #app .time-picker[open] .time-summary { border-color: var(--control-border); } +#app .time-summary:focus-visible { outline: 1px solid var(--accent); outline-offset: 3px; } +#app .time-shift { display: grid; place-items: center; width: 32px; height: 32px; padding: 0; line-height: 20px; } +#app .time-popover { position: absolute; z-index: 5; top: 38px; left: 0; right: 88px; display: grid; gap: 12px; padding: 14px; border: 1px solid var(--control-border); border-radius: var(--radius); background: var(--ground); } +#app .time-popover label { display: grid; align-items: center; grid-template-columns: 70px minmax(0, 1fr); gap: 8px; } +#app .time-popover :is(input, select) { width: 100%; height: 32px; } +#app .time-control input { min-width: 0; } +#app :is(.node-lane, .node-time-header) { display: grid; grid-template-columns: var(--node-label-width) minmax(0, 1fr) 48px; align-items: start; gap: 12px; } +#app .node-lane { padding: 10px 0; border-bottom: 1px solid var(--line); } +#app .node-time-header { padding-bottom: 8px; color: var(--dim); font-size: 11px; } +#app .node-axis { position: relative; display: flex; justify-content: space-between; margin-left: calc(var(--gpu-index-width) + var(--gpu-index-gap)); font-variant-numeric: tabular-nums; } +#app .node-axis span:nth-child(2) { position: absolute; left: 33.333333%; } +#app .node-axis span:nth-child(3) { position: absolute; left: 66.666667%; } +#app .node-axis span:not(:first-child):not(:last-child) { transform: translateX(-50%); } +#app .node-duty { text-align: right; font-variant-numeric: tabular-nums; line-height: 26px; } +#app .node-time-header .node-duty { line-height: inherit; } +#app .node-lane-label { display: flex; align-items: center; gap: 7px; min-width: 0; min-height: 26px; padding: 0; border: 0; background: transparent; text-align: left; white-space: nowrap; } +#app .node-lane-label:hover .node-accelerator { text-decoration: underline; text-underline-offset: 3px; } +#app .node-accelerator { min-width: 0; overflow: hidden; text-overflow: ellipsis; } +#app .unavailable-selection { display: flex; justify-content: space-between; align-items: center; gap: 12px; margin-bottom: 12px; color: var(--status-warning); } +#app .claim-label { flex: none; padding: 1px 5px; border-radius: 3px; background: var(--paper); color: var(--dim); font-size: 11px; line-height: 18px; white-space: nowrap; } +#app .gpu-capacity { display: grid; grid-template-columns: var(--gpu-index-width) minmax(0, 1fr); gap: var(--gpu-index-gap); min-width: 0; } +#app .gpu-lane-ids { display: grid; align-items: center; color: var(--dim); font: 11px var(--mono); } +#app .gpu-lane-ids span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#app .capacity-track { position: relative; min-width: 0; background: repeating-linear-gradient( to bottom, transparent 0 calc(var(--gpu-lane-height) - 1px), color-mix(in srgb, var(--line) 70%, transparent) calc(var(--gpu-lane-height) - 1px) var(--gpu-lane-height) ); } +#app :is(.capacity-track, .node-axis, .activity-track, .activity-axis, .allocation-expansion .chart-plot svg, .allocation-expansion .chart-empty) { touch-action: pan-y; cursor: grab; } +#app.node-navigating :is(.capacity-track, .node-axis, .activity-track, .activity-axis, .allocation-expansion .chart-plot svg, .allocation-expansion .chart-empty) { cursor: grabbing; user-select: none; } +#app .capacity-track::before { content: ""; position: absolute; inset: 0; pointer-events: none; background: repeating-linear-gradient( to right, transparent 0 calc(33.333333% - 1px), color-mix(in srgb, var(--line) 45%, transparent) calc(33.333333% - 1px) 33.333333% ); } +#app .hue-0 { --allocation-color: var(--hue-0); } +#app .hue-1 { --allocation-color: var(--hue-1); } +#app .hue-2 { --allocation-color: var(--hue-2); } +#app .hue-3 { --allocation-color: var(--hue-3); } +#app .hue-4 { --allocation-color: var(--hue-4); } +#app .hue-5 { --allocation-color: var(--hue-5); } +#app .hue-6 { --allocation-color: var(--hue-6); } +#app .hue-7 { --allocation-color: var(--hue-7); } +#app .capacity-allocation { container: allocation / inline-size; position: absolute; overflow: hidden; display: flex; align-items: center; justify-content: space-between; gap: 8px; min-width: 0; padding: 0; border: 0; border-radius: 2px; background: color-mix( in srgb, var(--allocation-color, var(--accent)) 19%, var(--ground) ); text-align: left; } +#app .capacity-allocation:hover { background: color-mix( in srgb, var(--allocation-color, var(--accent)) 27%, var(--ground) ); } +#app .capacity-allocation.selected:not(.held) { z-index: 1; background: color-mix( in srgb, var(--allocation-color, var(--accent)) 27%, var(--ground) ); outline: 1px solid color-mix(in srgb, var(--allocation-color, var(--accent)) 70%, var(--line)); } +#app .capacity-allocation.ended { --allocation-color: var(--dim); color: var(--dim); } +#app .capacity-allocation.held { overflow: hidden; background: color-mix(in srgb, var(--line) 45%, var(--ground)); } +#app :is(.allocation-name, .allocation-count) { position: relative; z-index: 1; } +#app .capacity-allocation.unknown-mapping { position: relative; width: 100%; height: 26px; margin-block: 3px; } +#app .allocation-name { margin-left: 7px; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#app .allocation-count { margin-right: 7px; flex: none; margin-left: auto; color: var(--dim); font-size: 11px; white-space: nowrap; } +@container allocation (max-width: 160px) { + #app :is(.allocation-name, .allocation-count) { display: none; } +} +#app .hold { position: absolute; top: 0; bottom: 0; background: color-mix(in srgb, var(--allocation-color) 35%, var(--ground)); pointer-events: auto; } +#app .hold:is(:hover, .selected) { background: color-mix(in srgb, var(--allocation-color) 65%, var(--ground)); } +#app .allocation-expansion { --activity-label-width: clamp(160px, 25vw, 240px); display: grid; grid-template-columns: minmax(0, 1fr); gap: 12px; margin: 0 calc(-1 * var(--gutter)); padding: 16px var(--gutter); border-bottom: 1px solid var(--line); } +#app .allocation-expansion > div { min-width: 0; } +#app .node-gpu-detail { display: grid; grid-template-columns: minmax(0, 1fr); gap: 12px; border-top: 1px solid var(--line); padding-top: 16px; } +#app .allocation-detail-head { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px; } +#app .allocation-detail-head h2 { font-size: 15px; font-weight: 500; line-height: 1.4; overflow-wrap: anywhere; } +#app .detail-close { display: grid; place-items: center; width: 28px; height: 28px; padding: 0; border: 1px solid transparent; border-radius: var(--radius); color: var(--dim); background: transparent; font-size: 22px; line-height: 1; } +#app .detail-close:hover { border-color: var(--line); background: var(--paper); color: var(--ink); } +#app .allocation-device-picker { display: flex; flex-wrap: wrap; gap: 4px; } +#app .allocation-device-picker .chip { border-color: transparent; background: transparent; color: var(--dim); } +#app .allocation-device-picker .chip:not([data-device="all"]) { min-width: 10ch; /* Fits "7 · 100%", so arriving and updating values never resize the chips. */ font-variant-numeric: tabular-nums; /* Equal-width digits: a new value of the same length doesn't move the text. */ text-align: center; } #app .allocation-device-picker .chip[aria-pressed="true"], #app .allocation-device-picker .chip:hover { border-color: var(--line); background: var(--paper); color: var(--ink); } #app .allocation-memory { color: var(--dim); font-size: 12px; } #app .allocation-memory strong { display: inline-block; min-width: 10ch; /* Fits "1536.0 GiB", so the label doesn't slide when the value arrives. */ margin-left: 8px; color: var(--ink); font-size: 15px; font-weight: 500; text-align: right; } +#app .allocation-detail-footer { display: flex; flex-wrap: wrap; justify-content: space-between; gap: 12px; color: var(--dim); font-size: 11px; } +#app :is(.activity-header, .activity-row) { display: grid; grid-template-columns: var(--activity-label-width) minmax(0, 1fr); column-gap: 12px; align-items: center; } +#app .activity-header { padding-block: 4px 10px; } +#app .activity-header h3 { margin: 0; font-size: 13px; font-weight: 500; } +#app .activity-axis { display: flex; justify-content: space-between; color: var(--dim); font: 11px var(--mono); } +#app .activity-row { min-height: 46px; border-top: 1px solid var(--line); } +#app .activity-label { display: flex; align-items: center; gap: 8px; min-width: 0; padding: 6px 8px; border: 1px solid transparent; border-radius: var(--radius); background: none; text-align: left; font-size: 12px; } +#app a.activity-label:hover { background: var(--hover); } +#app .activity-row[data-selected="true"] .activity-label { border-color: var(--line); background: var(--paper); } +#app .activity-swatch { flex: none; width: 9px; height: 9px; border-radius: 2px; background: color-mix(in srgb, var(--allocation-color) 65%, var(--ground)); } +#app .activity-label-text { min-width: 0; } +#app .activity-name { display: block; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#app .activity-meta { display: block; margin-top: 2px; color: var(--dim); font-size: 11px; } +#app .activity-track { position: relative; min-width: 0; height: 46px; display: flex; align-items: center; background: repeating-linear-gradient(to right, var(--line) 0 1px, transparent 1px 25%); box-shadow: inset -1px 0 var(--line); } +#app .activity-track svg { display: block; width: 100%; height: 24px; } +#app .activity-block { fill: color-mix(in srgb, var(--allocation-color) 35%, var(--ground)); cursor: default; } +#app .activity-block[data-placement] { cursor: pointer; } +#app .activity-block[data-selected="true"], #app .activity-block:hover { fill: color-mix(in srgb, var(--allocation-color) 65%, var(--ground)); } +#app .activity-block[data-selected="true"] { stroke: var(--allocation-color); stroke-width: 1px; } +/* A native SVG outline spans the gaps between intervals; stroke each interval instead. */ +#app .activity-block:focus { outline: none; } +#app .activity-block:focus-visible { stroke: var(--accent); stroke-width: 2px; } +/* Worker-recorded ops inside a GPU turn. The turn underneath stays visible + where no op was recorded. */ +#app [data-op] { --op-color: #2a9d8f; } +#app [data-op="training"], #app [data-op="forward_backward"] { --op-color: #3366cc; } +#app [data-op="forward"] { --op-color: #6fa8dc; } +#app [data-op="optim_step"] { --op-color: #8e44ad; } +#app [data-op="sample"] { --op-color: #2e9e5b; } +#app [data-op="weight_sync"] { --op-color: #e67e22; } +#app [data-op="wake_up"], #app [data-op="sleep"] { --op-color: #95a5a6; } +#app [data-op="init"] { --op-color: #5d6d7e; } +#app .activity-block.op { fill: var(--op-color); } +#app .activity-block.op:hover { fill: color-mix(in srgb, var(--op-color) 75%, var(--ink, #000)); } +#app .op-legend { display: flex; flex-wrap: wrap; gap: 4px 14px; margin: 8px 0 0 calc(var(--activity-label-width, 0px) + 12px); font-size: 12px; color: var(--muted, inherit); } +#app .op-legend > span { display: inline-flex; align-items: center; gap: 5px; } +#app .op-swatch { width: 10px; height: 10px; border-radius: 2px; background: var(--op-color); } +#app .op-swatch.turn { background: color-mix(in srgb, currentColor 25%, transparent); } +#app .cross-node-activity { --activity-label-width: clamp(160px, 25vw, 240px); border-top: 1px solid var(--line); padding-top: 16px; } +#app .cross-node-activity .activity-meta { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +/* The Run timeline has no pan gesture; a click opens the logs at that time. */ +#app section.cross-node-activity .activity-track { cursor: pointer; } +#app [data-crosshair] :is(.activity-track, .capacity-track)::after { content: ""; position: absolute; pointer-events: none; inset-block: 0; left: var(--crosshair); border-left: 1px dashed var(--dim); } +#app .activity-tooltip { position: fixed; z-index: 10; pointer-events: none; max-width: min(360px, calc(100vw - 16px)); max-height: calc(100dvh - 16px); overflow: hidden; overflow-wrap: anywhere; padding: 9px 12px; border: 1px solid var(--control-border); border-radius: var(--radius); background: var(--paper); font-size: 12px; } +#app .activity-tooltip-label { display: block; margin-bottom: 4px; } +#app :is(.activity-tooltip-time, .activity-tooltip-detail) { color: var(--dim); } +#app .activity-status { position: absolute; left: 8px; max-width: calc(100% - 16px); padding: 2px 4px; color: var(--dim); background: var(--ground); font-size: 11px; } +#app .activity-status.unavailable { color: var(--status-warning); } +/* Chart layout stays separate from the SVG gridlines. */ +#app div.chart-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 340px), 1fr)); gap: 24px 20px; margin-top: 16px; } +#app .chart { --metric-stroke: var(--dim); --plot-height: 160px; min-width: 0; min-height: calc(var(--plot-height) + 82px); } +#app .chart > .chart-empty { display: grid; place-items: center; height: var(--plot-height); margin: 0; } +#app .chart[data-tone="accent"] { --metric-stroke: color-mix(in srgb, var(--accent) 75%, var(--dim)); } +#app .allocation-expansion .chart { --plot-height: 100px; min-height: calc(var(--plot-height) + 82px); } +#app .allocation-expansion .chart-plot { grid-template-columns: var(--activity-label-width) minmax(0, 1fr); column-gap: 12px; } +#app .allocation-expansion .chart-empty { margin-left: calc(var(--activity-label-width) + 12px); } +#app .allocation-expansion .chart-line { stroke-width: 1.5; } +#app .chart-head { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 5px 16px; margin-bottom: 12px; } +#app .chart-head h2 { font-size: 13px; font-weight: 500; overflow-wrap: anywhere; } +#app .chart-latest { display: flex; align-items: baseline; gap: 7px; color: var(--dim); font-size: 11px; white-space: nowrap; } +#app .chart-latest strong { color: var(--ink); font: 500 15px var(--mono); } +#app .chart-plot { position: relative; display: grid; grid-template-columns: 44px minmax(0, 1fr); grid-template-rows: var(--plot-height) auto; column-gap: 8px; min-width: 0; } +#app .chart-y { margin-block: -6px; display: flex; flex-direction: column; justify-content: space-between; text-align: right; color: var(--dim); font: 11px var(--mono); } +#app .chart-plot:focus-visible { outline: 1px solid var(--control-border); outline-offset: 4px; } +#app .chart-plot svg { display: block; width: 100%; height: 100%; overflow: visible; border-bottom: 1px solid var(--line); } +#app .chart-x { grid-column: 2; position: relative; display: flex; justify-content: space-between; padding-top: 8px; color: var(--dim); font: 11px var(--mono); } +#app svg .chart-grid { stroke: var(--line); stroke-width: 1; } +#app .chart-line { fill: none; stroke: var(--metric-stroke); stroke-width: 1.6; stroke-linejoin: round; stroke-linecap: round; } +#app .chart-area { fill: var(--metric-stroke); fill-opacity: 0.07; } +#app .chart-band { fill: var(--allocation-color); fill-opacity: 0.55; } +#app .activity-block.wait { fill: var(--dim); fill-opacity: 0.16; } +#app .op-swatch.wait { background: var(--dim); opacity: 0.3; } +#app .chart-band.sampler { fill-opacity: 0.28; } +#app .chart-band-swatch { display: inline-block; width: 9px; height: 9px; margin-right: 5px; border-radius: 2px; background: var(--allocation-color); opacity: 0.55; vertical-align: -1px; } +#app .chart-band-key.sampler .chart-band-swatch { opacity: 0.28; } +#app .chart-dot { stroke: var(--metric-stroke); stroke-width: 5; stroke-linecap: round; } +#app .chart-hover { width: 0; position: absolute; top: 0; height: var(--plot-height); pointer-events: none; z-index: 2; } +#app .chart-hover[hidden] { display: none; } +#app .chart-cursor { position: absolute; inset: 0 auto 0 0; border-left: 1px dashed var(--control-border); } +#app .chart-crosshair { stroke: var(--control-border); stroke-dasharray: 3 3; } +#app .chart-tip { position: absolute; top: 6px; left: 8px; padding: 5px 8px; border: 1px solid var(--control-border); border-radius: var(--radius); background: var(--paper); color: var(--ink); font: 11px/1.5 var(--mono); white-space: nowrap; } +#app .chart-point { position: absolute; width: 6px; height: 6px; border-radius: 50%; background: var(--metric-stroke); transform: translate(-50%, -50%); } +#app .chart-summary { display: flex; flex-wrap: wrap; gap: 5px 18px; margin-top: 10px; color: var(--dim); font-size: 11px; } +#app .chart-summary strong { margin-left: 4px; color: var(--ink); font-family: var(--mono); font-weight: 400; } +#app .chart-note { color: var(--status-warning); } +#app .chart-empty { min-height: var(--plot-height); display: flex; align-items: center; color: var(--dim); } +/* Run details and logs. */ +#app .overview-back { margin-bottom: 16px; color: var(--dim); } +#app .run-heading { display: flex; align-items: center; flex-wrap: wrap; gap: 8px 16px; } +#app .run-description { margin-top: 4px; color: var(--dim); overflow-wrap: anywhere; } +#app .run-state { margin-top: 12px; overflow-wrap: anywhere; } +#app .run-toolbar { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 8px 16px; margin: 16px 0; padding-bottom: 12px; border-bottom: 1px solid var(--line); } +#app .run-time-controls label { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; } +#app .run-time-controls :is(label, a) { color: var(--dim); font-size: 12px; } +#app .run-incidents { display: grid; gap: 8px; margin-bottom: 16px; } +#app .run-incident { display: flex; align-items: baseline; flex-wrap: wrap; gap: 4px 12px; font-size: 12px; } +#app .incident-time { color: var(--dim); font-family: var(--mono); } +#app .incident-error .incident-label { color: var(--status-error); } +#app .incident-warning .incident-label { color: var(--status-warning); } +#app .incident-context { color: var(--dim); overflow-wrap: anywhere; } +#app .run-incident a { color: var(--dim); white-space: nowrap; } +#app .run-metric-summary { display: flex; flex-wrap: wrap; gap: 8px 16px; margin-bottom: 16px; } +#app .run-activity { border-top: 0; padding-top: 0; } +#app .run-activity .activity-title { margin-top: 8px; margin-bottom: 4px; } +#app .run-activity .activity-intro { margin: 0 0 12px; font-size: 12px; } +#app .run-activity :is(.activity-header, .activity-row) { grid-template-columns: var(--activity-label-width) 88px minmax(0, 1fr) 48px; } +#app .run-activity .activity-gpu { color: var(--dim); font: 11px var(--mono); overflow-wrap: anywhere; } +#app .activity-gpu-toggle { padding: 0; border: 0; background: none; text-align: left; cursor: pointer; } +#app .activity-gpu-toggle:hover { color: var(--ink); } +#app .activity-chevron { display: inline-block; width: 12px; } +#app .activity-duty { padding: 10px 0 14px; border-top: 1px solid var(--line); } +#app :is(.activity-mode, .chart-mode) { display: inline-flex; gap: 4px; } +#app .activity-mode { margin: 0 0 12px; } +#app .chart-mode { margin-left: 12px; margin-right: auto; } +#app .activity-duty .chart { --plot-height: 100px; min-height: calc(var(--plot-height) + 82px); } +/* The plot lines up with the lane tracks: past the label and GPU columns, short of the duty column. */ +#app .activity-duty .chart-plot { grid-template-columns: calc(var(--activity-label-width) + 100px) minmax(0, 1fr); column-gap: 12px; margin-right: 60px; } +#app .run-activity .activity-row[data-continued] { min-height: 30px; border-top-color: transparent; } +#app .run-activity .activity-row[data-continued] .activity-track { height: 30px; } +#app .run-activity .activity-row[data-lead] { min-height: 30px; } +#app .run-activity .activity-row[data-lead] .activity-track { height: 30px; } +/* A group's label overhangs the empty label cells of its later rows, so every GPU row keeps one pitch. */ +#app .run-activity .activity-row[data-lead] .activity-label { align-self: start; align-items: flex-start; height: 30px; overflow: visible; } +#app .activity-source { margin-top: 12px; color: var(--dim); font-size: 11px; } +#app .run-metrics { margin-top: 8px; } +#app .time-control.run-time-controls { grid-template-columns: 32px minmax(0, 1fr) 32px auto auto; width: 520px; } +#app .log-toolbar { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 12px; } +#app #run-panel > p + .log-toolbar { margin-top: 8px; } +#app .log-toolbar input { flex: 1; min-width: 100px; } +#app .log-toolbar select { max-width: min(100%, 340px); } +#app .log-toolbar .log-scope { display: flex; align-items: center; gap: 6px; color: var(--dim); font-size: 12px; white-space: nowrap; } +#app .log-toolbar .log-scope input { flex: none; min-width: 0; margin: 0; } +/* Fits "Jump to latest", so pausing by scrolling doesn't shift the toolbar. */ +#app .log-toolbar button { min-width: 8.75em; } +#app #log-status { display: grid; gap: 6px; color: var(--dim); font-size: 12px; } +#app #log-status:not(:empty) { margin-bottom: 12px; } +#app .source-error { margin: 8px 0; color: var(--status-warning); font-size: 12px; overflow-wrap: anywhere; } +#app .snapshot-error:not([hidden]) { display: flex; align-items: center; justify-content: space-between; gap: 8px 16px; margin: 16px var(--gutter) 0; padding: 12px; border: 1px solid color-mix(in srgb, var(--status-warning) 35%, var(--line)); border-radius: var(--radius); color: var(--status-warning); background: var(--paper); overflow-wrap: anywhere; } +#app .snapshot-error button { flex: none; } +#app pre, #app .workspace-logrow { white-space: pre-wrap; overflow-wrap: anywhere; font: 12px/1.65 var(--mono); } +/* At the bottom of the page the tabs, toolbar and lines fill the window; at the box's ends the wheel scrolls the page. */ +#app #log-lines { max-height: calc(100svh - 170px); overflow: auto; overflow-anchor: none; scrollbar-gutter: stable; } +#app .workspace-logrow { display: grid; grid-template-columns: 12ch var(--tag-width) minmax(0, 1fr); gap: 10px; padding: 1px 6px; border-left: 2px solid transparent; line-height: 1.5; } +#app .workspace-logrow[data-severity="error"] { border-left-color: var(--status-error); background: color-mix(in srgb, var(--status-error) 7%, transparent); } +#app .workspace-logrow[data-severity="warning"] { border-left-color: var(--status-warning); background: color-mix(in srgb, var(--status-warning) 7%, transparent); } +#app .log-time { color: var(--dim); white-space: nowrap; font-variant-numeric: tabular-nums; } +#app .log-pod { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--allocation-color, var(--dim)); } +#app .log-message { min-width: 0; margin: 0; font: inherit; } +#app .log-message mark { border-radius: 2px; background: color-mix(in srgb, var(--hue-3) 30%, transparent); color: inherit; } +#app .log-older { margin: 0 0 8px; } +#app .log-error { color: var(--status-error); } +#app .log-time-scope { display: flex; align-items: baseline; flex-wrap: wrap; gap: 8px 16px; margin-bottom: 12px; color: var(--dim); font-size: 12px; } +#app .log-time-scope a { color: var(--ink); } +#app .run-json-link { margin-top: 20px; color: var(--dim); font-size: 12px; } +/* Section titles and wide tables on the Run and Health pages. */ +#app .scheduler-title { margin: 24px 0 12px; font-size: 15px; font-weight: 500; line-height: 1.4; } +#app :is(.scheduler-table-wrap, .table-scroll) { overflow-x: auto; } +#app .scheduler-table { min-width: 650px; } +/* Health: only the evidence wraps, so pod names and links stay whole. */ +#app .scheduler-table :is(th, td):not(:nth-child(3)) { white-space: nowrap; } +/* Keep the identity readable before switching to compact rows. */ +@media (min-width: 761px) and (max-width: 1100px) { + #app :is(.job-list-head, .job-list-row).run-row { grid-template-columns: minmax(0, 1fr) 112px 56px 72px 140px; gap: 16px; } +} +@media (max-width: 760px) { + #app :is(.job-list-head, .job-list-row).run-row { grid-template-columns: minmax(0, 1fr) minmax(100px, auto); gap: 6px 12px; } + #app .job-list-head.run-row > :nth-child(n + 3) { display: none; } + #app .run-row > :nth-child(2) { justify-self: end; } + #app .run-row > :nth-child(n + 3) { color: var(--dim); font-size: 12px; } + #app .run-row > :nth-child(3) { grid-column: 1; text-align: left; } + #app .run-row > :nth-child(4) { grid-column: 2; text-align: right; } + #app .job-list-row.run-row > :nth-child(3)::after { content: " steps"; } + #app .job-list-row.run-row > :nth-child(4)::before { content: "Elapsed "; } + #app .job-list-row.run-row > .run-session { grid-column: 1 / -1; text-align: left; } +} +@media (max-width: 580px) { + #app .time-control { width: 100%; } + #app { --gutter: 14px; --gpu-index-width: 24px; --gpu-index-gap: 4px; } + #app .appbar { padding-block: 12px; gap: 8px 16px; } + #app .appbar > nav { order: 3; width: 100%; flex-wrap: nowrap; gap: 16px; overflow-x: auto; white-space: nowrap; font-size: 12px; } + #app #run-search { max-width: none; } + #app .pad { padding-block: 16px; } + #app .heading { font-size: 20px; } + #app :is(.node-lane, .node-time-header) { grid-template-columns: minmax(0, 1fr) 36px; gap: 6px 8px; } + #app .node-time-header > span:first-child { display: none; } + #app .node-lane-label { grid-column: 1 / -1; min-height: 20px; gap: 6px; font-size: 12px; } + #app .time-control { grid-template-columns: 32px minmax(0, 1fr) 32px; } + #app .time-control > [data-copy-view] { grid-column: 1 / -1; justify-self: end; } + #app .time-popover { right: 0; } + #app .claim-label { padding-inline: 3px; font-size: 10px; } + #app .node-axis span:not(:first-child):not(:last-child), #app .chart-x span:not(:first-child):not(:last-child) { display: none; } + #app .capacity-track::before { background: none; border-right: 1px solid var(--line); } + #app :is(.allocation-expansion, .cross-node-activity) { --activity-label-width: 40px; } + #app :is(.activity-header, .activity-row) { grid-template-columns: minmax(0, 1fr); } + #app .activity-header h3 { display: none; } + #app .activity-label { margin-top: 4px; } + #app .activity-track { height: 34px; } + #app .op-legend { margin-left: 0; } + #app .activity-axis span:not(:first-child):not(:last-child) { display: none; } + #app .run-activity :is(.activity-header, .activity-row) { grid-template-columns: 64px minmax(0, 1fr) 36px; } + #app .run-activity .activity-label { grid-column: 1 / -1; } + #app .run-activity .activity-axis { grid-column: 2; } + #app .run-activity .activity-row > span:empty { display: none; } + #app .chart-tip { max-width: 230px; white-space: normal; width: max-content; } + #app .run-toolbar { align-items: flex-start; } + #app .run-time-controls { width: 100%; justify-content: space-between; } + #app .log-toolbar > :is(input, select) { flex: 1 1 100%; width: 100%; min-width: 0; max-width: 100%; } + #app .workspace-logrow { grid-template-columns: auto minmax(0, 1fr); gap: 0 10px; padding-block: 4px; } + #app .log-message { grid-column: 1 / -1; } +} diff --git a/src/server/observability/static/views.js b/src/server/observability/static/views.js new file mode 100644 index 000000000..75572a88c --- /dev/null +++ b/src/server/observability/static/views.js @@ -0,0 +1,461 @@ +// Overview, Health, Nodes, and Run (Activity, Metrics, Logs) views. +import { renderActivity } from "./activity.js"; +import { renderChart, timeControlHtml } from "./charts.js"; +import { KINDS, elapsedTime, escape, escapeRegExp, finishedAt, isFinished, runDescription, runLabel, shortNodeName, statusBadge } from "./format.js"; +import { MAX_WINDOW, get, inspectUrl, now, rangeOf, saved, ui, use, view, viewLink } from "./state.js"; +import { DCGM_READING_SECONDS, acceleratorLabel, activitySource, adaptLanes, axisTime, capacityBars, claimLabel, clipPlacements, deviceLabel, deviceSeries, dutyBands, gapFor, lastValue, opLabel, opRank, placementColor } from "./visualization.js"; + +const OVERVIEW_FILTERS = [ + ["all", "All", "Every recorded run"], + ["active", "Active", "Runs whose workers are placed, or whose placement cannot be read"], + ["queued", "Queued", "Waiting for a worker to be placed, or for the server to create one"], + ["finished", "Finished", "Completed, or nothing serves the run and its session is gone"], +]; +const runGroup = (run) => ({ ended: "finished", waiting: "queued" })[run.lifecycle] || "active"; +const runLink = (run, tab = "activity") => viewLink(`run/${encodeURIComponent(run.run_id)}/${tab}`); +const healthStatus = (label, tone) => `${escape(label)}`; + +export function renderOverview(inventory) { + const summaries = use(inspectUrl("runs", { ended: "true" }), { scope: "overview:runs", poll: 15000 }); + const stepsByRun = new Map((summaries.data?.runs || []).map((r) => [r.run_id, r.steps])); + const terms = view.q.toLowerCase().split(/\s+/).filter(Boolean); + const shown = inventory.runs.filter((run) => { + const text = [run.run_id, run.name, run.model, run.recipe_name, runLabel(run), KINDS[run.fine_tuning_type], ...run.nodes, ...run.pods.map((p) => p.name)].filter(Boolean).join(" ").toLowerCase(); + return (view.status === "all" || runGroup(run) === view.status) && terms.every((term) => text.includes(term)); + }).sort((a, b) => Number(runGroup(b) === "active") - Number(runGroup(a) === "active")); + + const filters = OVERVIEW_FILTERS.map(([key, label, title]) => { + const count = key === "all" ? inventory.runs.length : inventory.runs.filter((r) => runGroup(r) === key).length; + return ``; + }).join(""); + + const rows = shown.map((run) => { + const label = runDescription(run, null); + const model = (run.model || "Run").split("/").at(-1); + const kind = KINDS[run.fine_tuning_type] || run.fine_tuning_type; + const session = run.session_ids?.at(-1) ?? null; + const steps = run.steps ?? stepsByRun.get(run.run_id) ?? "—"; + return `
+ ${escape(model)} · ${escape(run.run_id.slice(0, 8))}${kind ? escape(` · ${kind}`) : ""}${label ? `${escape(label)}` : ""} + ${statusBadge(runLabel(run))}${escape(steps)}${escape(elapsedTime(run, inventory.observed_at))} + ${escape(session ?? "—")} +
`; + }).join(""); + + return `

Overview

+
+ +
${filters}
+
+
${shown.length ? `
JobStatusStepsElapsedSession
${rows}` : ""}
+ ${!shown.length ? `
${inventory.runs.length ? "No runs match the search and status filter" : "No runs recorded"}
` : ""}`; +} + +export function renderHealth(inventory) { + const errors = [...new Set(inventory.errors)]; + const pods = inventory.cluster.pods.map((pod) => { + const run = inventory.runs.find((r) => r.pods.some((p) => p.name === pod.name)); + const [reason, detail = ""] = pod.problem.split(/:\s*(.*)/s); + const evidence = [detail, pod.restarts ? `${pod.restarts} restarts` : ""].filter(Boolean).join(" · ") || "—"; + const tone = ["Pending", "Unschedulable"].includes(reason) ? "warning" : "error"; + return { issue: reason, resource: pod.name, evidence, tone, link: run && runLink(run, "logs"), label: "Logs" }; + }); + const nodes = (inventory.cluster.nodes || []).filter((n) => n.ready !== true).map((n) => ({ + issue: "Node not ready", resource: n.name, evidence: "Ready condition is false or unknown", tone: "error", link: viewLink("nodes", { node: n.name }), label: "Nodes", + })); + const issues = [...pods, ...nodes]; + const errRows = errors.map((e) => `

${healthStatus("Source unavailable", "warning")}${escape(e)}

`).join(""); + const table = issues.length + ? `
${issues.map((i) => ``).join("")}
IssueResourceEvidence
${healthStatus(i.issue, i.tone)}${escape(i.resource)}${escape(i.evidence)}${i.link ? `${escape(i.label)} ↗` : ""}
` + : inventory.cluster.available === true && !errors.length + ? `

${healthStatus("Healthy", "success")}No pod problems, node problems or source errors.

` + : ""; + return `

Health

${errRows}${table}`; +} + +export function renderNodes(inventory) { + const observed = now(inventory); + const range = rangeOf(inventory, view); + const duration = range.now - range.start; + if (view.duration !== duration) view.duration = duration; + if (view.end !== null && view.end !== range.now) view.end = range.now; + + const bounds = { since: new Date(range.start * 1000).toISOString(), until: new Date(range.now * 1000).toISOString() }; + const live = view.end === null; + const queryOpts = { poll: 15000, pollWhen: live, remember: !live }; + const history = use(inspectUrl("activity", bounds), { ...queryOpts, scope: "activity:nodes" }); + const fleet = use(inspectUrl("query", { name: "gpu_busy", points: 1, ...bounds }), { ...queryOpts, scope: "query:fleet-busy" }); + + const busyByGpu = Object.fromEntries((fleet.data?.series || []).filter((s) => s.points.length).map((s) => [s.labels.UUID, s.points.at(-1)[1]])); + const dutyCycle = (n) => { + const values = (n.devices || []).map((d) => busyByGpu[d.id]).filter(Number.isFinite); + return values.length ? `${Math.round(values.reduce((a, b) => a + b, 0) / values.length)}%` : "—"; + }; + + const loading = !history.data && !history.error; + const all = clipPlacements(adaptLanes(history.data?.lanes || [], inventory.runs), range, history.error, history.status); + const selectedPlacement = all.find((p) => p.id === view.placement); + if (!view.node && selectedPlacement) view.node = selectedPlacement.node; + const node = inventory.cluster.nodes.find((n) => n.name === view.node); + if (node && selectedPlacement && selectedPlacement.node !== node.name) view.placement = null; + if (node && (node.devices.length === 1 || (view.gpu !== "all" && !node.devices.some((d) => d.id === view.gpu)))) view.gpu = "all"; + + const utilQuery = use(node ? inspectUrl("query", { name: "gpu_util", node_id: node.name, ...bounds }) : null, { ...queryOpts, scope: `node-util:${node?.name || ""}` }); + const memQuery = use(node ? inspectUrl("query", { name: "gpu_memory_mib", node_id: node.name, ...bounds }) : null, { ...queryOpts, scope: `node-mem:${node?.name || ""}` }); + + const errors = [...new Set([...inventory.errors, history.error].filter(Boolean))]; + const busy = (n) => Number(inventory.placements.some((p) => p.node === n.name)); + const nodes = inventory.cluster.nodes.filter((n) => n.gpu_capacity > 0 || n.devices?.length).sort((a, b) => busy(b) - busy(a)); + const unavailable = !node && (view.node || (view.placement && history.data)); + const back = viewLink("nodes", { duration, end: range.now, node: view.node, placement: view.placement, gpu: view.node && view.gpu !== "all" ? view.gpu : null }); + + const axis = [0, 1, 2, 3].map((step) => `${escape(axisTime(range.start + (range.now - range.start) * step / 3, range))}`).join(""); + + const groupsHtml = loading + ? `

Loading activity…

` + : !nodes.length + ? (!errors.length && inventory.cluster.available ? `

No accelerator nodes reported by Kubernetes

` : "") + : nodes.map((item) => { + const devices = item.devices || []; + const height = devices.length === 1 ? 30 : 22; + const placements = all.filter((p) => p.node === item.name && p.start <= range.now && p.end >= range.start); + const current = range.live ? inventory.placements.filter((p) => p.node === item.name) : placements.filter((p) => p.start <= range.now && p.end >= range.now); + const unmapped = placements.filter((p) => !p.devices.length || p.devices.some((id) => !devices.some((d) => d.id === id))); + + const bars = capacityBars(devices, placements, range, view.placement).map((bar) => { + const holds = bar.holds.map((h) => ``).join(""); + return ``; + }).join(""); + + const track = devices.length + ? `
${devices.map((d) => `${escape(deviceLabel(d.name))}`).join("")}
${bars}
` + : ""; + const unmappedHtml = unmapped.map((p) => ``).join(""); + const noteHtml = unmapped.length ? `Device mapping unavailable` : !devices.length && item.gpu_capacity ? `

GPUs without DRA devices

` : ""; + + let detailHtml = ""; + if (item.name === view.node) { + const found = new Map(); + for (const [field, q] of [["utilization", utilQuery], ["memory_mib", memQuery]]) { + for (const s of q.data?.series || []) found.set(s.labels.UUID, { id: s.labels.UUID, ...found.get(s.labels.UUID), [field]: s.points }); + } + const metricDevices = [...found.values()]; + const loaded = Boolean(utilQuery.data || utilQuery.error || memQuery.data || memQuery.error); + const selectedDevices = metricDevices.filter((d) => view.gpu === "all" || view.gpu === d.id); + const gpuPoints = deviceSeries(selectedDevices, "utilization"); + const gpuMemory = lastValue(deviceSeries(selectedDevices, "memory_mib"), range); + const gpuError = utilQuery.error || utilQuery.data?.error || ""; + const gpuMemoryError = memQuery.error || memQuery.data?.error || ""; + const visible = all.filter((p) => p.node === item.name && (view.gpu === "all" || p.devices.includes(view.gpu)) && p.start <= range.now && p.end >= range.start); + const label = `${acceleratorLabel(item)} · ${shortNodeName(item.name, inventory.cluster.nodes)}`; + + const picker = devices.length > 1 + ? `
${devices.map((d) => { + const v = lastValue(deviceSeries(metricDevices.filter((m) => m.id === d.id), "utilization"), range); + return ``; + }).join("")}
` + : ""; + + const gpuDetail = item.gpu_capacity + ? `
${renderChart({ + id: `node-duty:${item.name}:${view.gpu}`, + title: `Duty cycle · ${label}`, + unit: "%", + points: gpuPoints, + start: range.start, + end: range.now, + min: 0, + max: 100, + readingSeconds: DCGM_READING_SECONDS, + bands: dutyBands(visible), + gapSeconds: gapFor(gpuPoints, 60), + empty: gpuError || (!loaded && "Loading…") || "No samples for this GPU", + note: gpuError ? `${gpuError} · Showing previously fetched samples` : "", + })}${utilQuery.error ? `` : ""}
+
` + : ""; + + detailHtml = `
+

${escape(label)}

+ ${picker} +
${renderActivity({ placements: visible, range, inventory, selected: view.placement, selectable: true, back, activeOnly: true, showSource: false })}
+ ${gpuDetail} +
`; + } + + return `
+
+ +
${track}${unmappedHtml}${noteHtml}
+ ${escape(dutyCycle(item))} +
+ ${detailHtml} +
`; + }).join(""); + + return `
+

Accelerator nodes

${timeControlHtml(range, observed)}
+ ${errors.map((e) => `

${escape(e)}

`).join("")} + ${history.error ? '' : ""} + ${!inventory.cluster.available && !errors.length ? '

Kubernetes unavailable

' : ""} + ${unavailable ? `
${view.node ? "The selected node is no longer reported." : "The selected placement is not in retained history."}
` : ""} +
Node
${axis}
Duty
+ ${groupsHtml} +
`; +} + +// --- Run Logs state machine (tailing, pause, older cursor paging, search debounce) --- +export const logsState = { + data: null, + pending: false, + error: null, + everything: false, + filterKey: null, + queryKey: null, + lastQ: "", + failedCursor: null, + fromBottom: 0, + needsScroll: false, + controller: null, + timer: null, + params: null, + rangeNow: 0, +}; + +export function stopLogs() { + clearTimeout(logsState.timer); + logsState.timer = null; + logsState.controller?.abort(); + logsState.controller = null; + logsState.pending = false; +} + +export async function loadLogs(query, page = null) { + stopLogs(); + const controller = logsState.controller = new AbortController(); + const windowEnd = logsState.rangeNow; + logsState.pending = true; + logsState.failedCursor = page; + ui.render(); + try { + const { data: response } = await get(inspectUrl("logs", { ...query, ...(page ? { cursor: page } : {}) }), controller.signal); + if (controller.signal.aborted) return; + if (response.error) throw new Error(response.error); + const lines = document.getElementById("log-lines"); + logsState.fromBottom = page && lines ? lines.scrollHeight - lines.clientHeight - lines.scrollTop : 0; + const prev = page ? logsState.data?.records || [] : []; + const merged = [...new Map([...prev, ...(response.records || [])].map((r) => [r.id, r])).values()].slice(0, 2000); + logsState.data = { ...response, queryKey: logsState.queryKey, windowEnd, records: merged }; + logsState.error = null; + logsState.needsScroll = true; + } catch (failure) { + if (!controller.signal.aborted) { + logsState.error = failure.message; + if (page && failure.status === 400) logsState.failedCursor = null; + } + } finally { + if (!controller.signal.aborted) { + logsState.pending = false; + logsState.controller = null; + if (view.page === "run" && view.tab === "logs" && view.end === null) { + logsState.timer = setTimeout(() => { + if (!document.hidden && !logsState.pending && view.page === "run" && view.tab === "logs" && view.end === null && logsState.params) { + loadLogs(logsState.params); + } + }, 5000); + } + ui.render(); + } + } +} + +export function pauseLogs() { + stopLogs(); + view.end = logsState.data?.windowEnd ?? logsState.rangeNow; + if (logsState.data) logsState.data = { ...logsState.data, queryKey: `${logsState.filterKey}:${view.end}` }; + ui.render(); +} + +export function followLogs() { + view.event = view.end = null; + if (logsState.data) logsState.data = { ...logsState.data, next_cursor: null, queryKey: null }; + ui.render(); +} + +function renderRunLogs(run, range, allLogs) { + const scope = view.q || (run.shared_runtime && !logsState.everything ? run.run_id : ""); + const logRange = { + since: new Date(Math.max(range.start, view.event === null ? range.start : view.event - 120) * 1000).toISOString(), + until: new Date(Math.min(range.now, view.event === null ? range.now : view.event + 120) * 1000).toISOString(), + }; + const params = { run_id: view.id, q: scope, limit: "200", ...logRange, ...(view.pod ? { pod: view.pod } : {}) }; + const filterKey = JSON.stringify([view.id, scope, view.pod, view.event, view.duration]); + const queryKey = `${filterKey}:${view.end}`; + logsState.params = params; + logsState.rangeNow = range.now; + + if (!ui.navigating && queryKey !== logsState.data?.queryKey && queryKey !== logsState.queryKey) { + const changed = filterKey !== logsState.filterKey; + const searched = changed && logsState.filterKey !== null && params.q !== logsState.lastQ; + if (changed) { logsState.data = logsState.error = null; } + logsState.filterKey = filterKey; + logsState.queryKey = queryKey; + logsState.lastQ = params.q; + stopLogs(); + logsState.pending = true; + logsState.timer = setTimeout(() => loadLogs(params), searched ? 250 : 0); + } + + const records = logsState.data?.records || []; + const cursor = records.length < 2000 ? logsState.data?.next_cursor : null; + const pods = new Map((run.pods || []).map((p) => [p.name, p])); + for (const r of [...(logsState.data?.sources || []), ...records]) if (r.pod && !pods.has(r.pod)) pods.set(r.pod, { ...r, name: r.pod }); + if (view.pod && !pods.has(view.pod)) pods.set(view.pod, { name: view.pod }); + if (!pods.has("client")) pods.set("client", { name: "client" }); + const sources = [...pods.values()].sort((a, b) => (a.name === "client") - (b.name === "client") || a.name.localeCompare(b.name)); + + const [first = "", ...rest] = sources.map((p) => p.name).filter((name) => name !== "client"); + let cutLen = 0; + while (rest.length && cutLen < first.length && rest.every((name) => name[cutLen] === first[cutLen])) cutLen++; + const cut = rest.length ? first.lastIndexOf("-", cutLen - 1) + 1 : first.length; + const hues = new Map(sources.map((p, i) => [p.name, `hue-${i % 8}`])); + const tag = (name) => name === "client" ? name : name?.slice(cut); + const tagWidth = Math.min(20, Math.max(0, ...sources.map((p) => tag(p.name).length))); + const match = view.q ? new RegExp(`(${escapeRegExp(view.q)})`, "i") : null; + const shown = records.toReversed(); + const paging = logsState.pending && view.end !== null; + const sourceName = logsState.data?.source?.startsWith("cloud-logging") ? "Cloud Logging" : logsState.data?.source || "Logs"; + const status = [ + sourceName, + records.length || !logsState.error ? `${records.length.toLocaleString()} ${records.length === 1 ? "record" : "records"}` : "", + logsState.pending && !records.length ? "Loading…" : logsState.error ? "" : view.end === null ? "Live" : "Paused", + ].filter(Boolean).join(" · "); + + const formatMsg = (msg) => match + ? msg.split(match).map((part, i) => i % 2 ? `${escape(part)}` : escape(part)).join("") + : escape(msg); + const sev = (r) => ["ERROR", "CRITICAL", "ALERT", "EMERGENCY"].includes(r.severity) ? "error" : r.severity === "WARNING" ? "warning" : ""; + + const rowsHtml = shown.length + ? shown.map((r) => { + const at = Number.isFinite(Date.parse(r.timestamp)) ? new Date(r.timestamp).toISOString() : ""; + const s = sev(r); + return `
${escape(tag(r.pod))}
${formatMsg(r.message)}${r.message_truncated ? '\n[Message truncated by source]' : ""}
`; + }).join("") + : (!logsState.pending && !logsState.error ? `

${escape(logsState.data?.note || "No logs match this time range and filter")}

` : ""); + + return `${view.event !== null ? `
${escape(`${logRange.since.slice(0, 10)} · ${logRange.since.slice(11, 19)}–${logRange.until.slice(11, 19)}`)}All logs
` : ""} +
+ + + ${run.shared_runtime ? `` : ""} + +
+
${escape(status)}${logsState.error ? `${escape(logsState.error)}${records.length ? " · Showing previously fetched records" : ""}` : ""}
+
+ ${cursor ? `` : records.length === 2000 ? '

2,000 records shown. Narrow the time range or search to inspect more.

' : ""} + ${rowsHtml} +
`; +} + +function renderRunMetrics(run, finished) { + const steps = use(inspectUrl(`runs/${encodeURIComponent(view.id)}/steps`, { last: 1000 }), { scope: `steps:${view.id}`, poll: 30000, pollWhen: !finished }); + const rows = steps.data?.steps || []; + const loading = !steps.data && !steps.error; + const stepRange = rows.length ? { start: rows[0].step, end: rows.at(-1).step } : null; + const completedSteps = steps.data ? rows.filter((r) => r.complete).length : run.steps ?? "—"; + const byStep = (fn) => rows.map((row) => [row.step, fn(row)]).filter(([, v]) => Number.isFinite(v)); + + const trainerCharts = [...new Map(rows.flatMap((row) => Object.entries(row.ops).flatMap(([op, entry]) => Object.keys(entry.metrics).filter((name) => !name.includes(":") || name.endsWith(":mean")).map((name) => [`${op} ${name}`, [op, name]])))).values()] + .sort(([a, x], [b, y]) => opRank(a) - opRank(b) || x.localeCompare(y)); + const opNames = [...new Set(rows.flatMap((row) => Object.keys(row.ops)))].sort((a, b) => opRank(a) - opRank(b) || a.localeCompare(b)); + const placeholders = (n, prefix) => `
${Array.from({ length: n }, (_, i) => renderChart({ id: `${prefix}-${i}`, title: "", points: [], empty: "Loading…" })).join("")}
`; + + const trainerHtml = trainerCharts.length + ? `
${trainerCharts.map(([op, name]) => renderChart({ id: `tm:${op}:${name}`, title: `${name} · ${opLabel(op)}`, steps: true, points: byStep((r) => r.ops[op]?.metrics[name]?.mean), start: stepRange.start, end: stepRange.end, tone: "accent", empty: "No values" })).join("")}
` + : loading ? placeholders(3, "tm-load") : steps.data ? '

0 metrics

' : ""; + const opsHtml = opNames.map((op) => renderChart({ id: `op:${op}`, title: `${opLabel(op)} duration`, unit: "s", steps: true, points: byStep((r) => r.ops[op] && r.ops[op].seconds / r.ops[op].count), start: stepRange.start, end: stepRange.end, min: 0, tone: "accent", empty: "No values" })).join(""); + + return `
+ ${steps.error ? `

${escape(steps.error)}${rows.length ? " · Showing previously fetched steps" : ""}

` : ""} +

Trainer metrics

${trainerHtml} +

Operations

+
Completed steps ${escape(completedSteps)}
+ ${loading ? placeholders(3, "op-load") : ""} +
${opsHtml}${steps.data && !rows.length ? '

No operations recorded

' : ""}
+
`; +} + +function setupOf(s) { + const devices = s["openrl.trainer_device_count"] ?? s["openrl.trainer_gpus"], cp = s["openrl.trainer_cp"], samplers = s["openrl.sampler_replicas"]; + return [ + { automodel: "Automodel", pytorch: "PyTorch" }[s["openrl.trainer_backend"]] || s["openrl.trainer_backend"], + devices != null && `${devices} trainer device${devices === 1 ? "" : "s"}${cp > 1 ? ` (CP ${cp})` : ""}`, + samplers != null && `${samplers} sampler${samplers === 1 ? "" : "s"}`, + s["openrl.exclusive"] && "exclusive GPUs", + s["openrl.fine_tuning_type"] === "full" && s["openrl.weight_sync"] && `${s["openrl.weight_sync"]} weight sync`, + ]; +} + +export function renderRun(inventory) { + const run = inventory.runs.find((r) => r.run_id === view.id); + const observed = now(inventory); + const finished = isFinished(run); + if (run && !ui.runOpened.has(view.id)) { + ui.runOpened.add(view.id); + const stop = finishedAt(run); + if (finished && !view.explicitWindow && view.end === null && view.duration === 1800 && Number.isFinite(stop)) { + const span = Math.max(0, stop - (run.created_at || stop)); + view.duration = Math.max(1800, Math.min(MAX_WINDOW, Math.round(span * 1.05))); + view.end = Math.min(observed, Math.round(stop + Math.max(60, span * 0.025))); + } + } + + const range = rangeOf(inventory, view); + if (view.event !== null && (view.event < range.start || view.event > range.now)) view.event = null; + const tab = ["metrics", "logs"].includes(view.tab) ? view.tab : "activity"; + if (tab !== "logs") stopLogs(); + + const bounds = { since: new Date(range.start * 1000).toISOString(), until: new Date(range.now * 1000).toISOString(), run_id: view.id }; + const activityUrl = inspectUrl("activity", bounds); + const live = view.end === null; + const detail = use(tab === "activity" ? activityUrl : null, { scope: `run-activity:${view.id}`, poll: 15000, pollWhen: live && tab === "activity", remember: !live }); + + const backHtml = `

← ${view.back ? "Nodes" : "Overview"}

`; + if (!run) { + return `${backHtml}

Run

${detail.error ? `

${escape(detail.error)}

` : `

${view.id ? "Loading run…" : "Choose a run from Overview"}

`}`; + } + + const title = [(run.model || "Run").split("/").at(-1), run.run_id.slice(0, 8), KINDS[run.fine_tuning_type]].filter(Boolean).join(" · "); + const description = [runDescription(run, title), ...setupOf(run.settings || {})].filter(Boolean).join(" · "); + const link = (t, changes = {}) => viewLink(`run/${encodeURIComponent(view.id)}/${t}`, { duration: view.duration, end: view.end, back: view.back, event: view.event, q: view.q, pod: view.pod, ...changes }); + + let panel; + if (tab === "logs") { + panel = renderRunLogs(run, range, link("logs", { event: null, end: null })); + } else if (tab === "metrics") { + panel = renderRunMetrics(run, finished); + } else { + const busy = use(inspectUrl("query", { name: "gpu_busy", ...bounds, points: 1 }), { scope: `run-busy:${view.id}`, poll: 15000, pollWhen: live, remember: !live }); + const duty = Object.fromEntries((busy.data?.series || []).filter((s) => s.points.length).map((s) => [s.labels.UUID, Math.round(s.points.at(-1)[1])])); + const slots = Object.fromEntries((busy.data?.series || []).filter((s) => s.labels.gpu != null).map((s) => [s.labels.UUID, s.labels.gpu])); + const clipped = clipPlacements(adaptLanes(detail.data?.lanes || [], inventory.runs, view.id), range, detail.error, detail.status); + panel = `
${renderActivity({ placements: clipped, range, inventory, acrossNodes: true, duty, slots, error: detail.error, pending: detail.pending, pickLogs: true })}
`; + } + + const problems = (run.pods || []).filter((p) => p.problem).map((p) => `

${escape(`${p.name}: ${p.problem}`)}

`).join(""); + return `${backHtml} +

${escape(title)}

${statusBadge(runLabel(run))}
+ ${detail.error ? `

${escape(detail.error)}

` : ""} + ${description ? `

${escape(description)}

` : ""} +
${problems}${!finished && run.shared_runtime ? '

Shared runtime: its workers also serve other runs

' : ""}
+
+ +
${timeControlHtml(range, observed)}Agent JSON ↗
+
+
${panel}
`; +} diff --git a/src/server/observability/static/visualization.js b/src/server/observability/static/visualization.js new file mode 100644 index 000000000..117fe2b03 --- /dev/null +++ b/src/server/observability/static/visualization.js @@ -0,0 +1,179 @@ +// Presentation geometry and inspection API normalization. +import { family } from "./format.js"; +export { shortNodeName } from "./format.js"; + +const numbers = ["standard", "compact"].map((notation) => new Intl.NumberFormat(undefined, { maximumSignificantDigits: 4, notation })); +const percent = new Intl.NumberFormat(undefined, { maximumFractionDigits: 1 }); +export const chartNumber = (value) => numbers[Number(Math.abs(value) >= 10000)].format(value); +export const chartTime = (at, seconds = false) => new Date(at * 1000).toISOString().slice(11, seconds ? 19 : 16); +export const valueText = (value, unit) => unit === "%" ? `${percent.format(value)}%` : `${chartNumber(value)}${unit ? ` ${unit}` : ""}`; +export const axisTime = (at, range) => `${range.now - range.start >= 86400 ? `${new Date(at * 1000).toISOString().slice(5, 10)} ` : ""}${chartTime(at, range.now - range.start <= 120)}`; +export const placementColor = (p) => family(p.run_ids?.[0] || p.runtime_id); +// The exporter's collection interval in k8s/deploy/dcgm; each engine-activity reading averages it. +export const DCGM_READING_SECONDS = 10; + +export function adaptInventory(raw) { + if (!raw) return raw; + const nodes = (raw.cluster?.nodes || []).map((n) => ({ + ...n, + devices: n.devices || (n.gpus || []).map((g) => ({ id: g.uuid || g.name, name: g.name, product: g.product, memory: g.memory })), + })); + const placements = (raw.placements || []).map((p) => { + const devices = p.devices || p.gpu_uuids || []; + return { ...p, devices, device_count: p.device_count ?? devices.length }; + }); + return { ...raw, cluster: { ...raw.cluster, nodes }, placements }; +} + +function mergeSpans(spans) { + const merged = []; + for (const [a, b] of spans.toSorted((p, q) => p[0] - q[0] || p[1] - q[1])) { + if (merged.length && a <= merged.at(-1)[1]) merged.at(-1)[1] = Math.max(merged.at(-1)[1], b); + else merged.push([a, b]); + } + return merged; +} + +function laneLabel(lane, byId, runId) { + if (runId && byId.has(runId)) return byId.get(runId).name; + const ids = lane.run_ids || []; + if (ids.length === 1) return byId.get(ids[0])?.name || ids[0].slice(0, 8); + if (ids.length > 1) { + const model = byId.get(ids[0])?.model?.split("/").at(-1); + return model ? `${model} (${ids.length} runs)` : `${ids.length} runs`; + } + return lane.name; +} + +export function adaptLanes(lanes = [], runs = [], runId = null) { + const byId = new Map((runs || []).map((r) => [r.run_id, r])); + return lanes.map((lane) => { + const held = lane.intervals || []; + const devices = lane.gpu_uuids || []; + const turns = lane.turns || []; + const waits = lane.waits || []; + const active = runId ? (lane.active?.[runId] || []) : mergeSpans(Object.values(lane.active || {}).flat()); + const exact = turns.length > 0; + const intervals = exact ? turns : active.length ? active : held; + const opName = lane.role === "sampler" ? "sample" : "training"; + return { + ...lane, + id: `${lane.name}:${lane.node || ""}:${devices.join(",")}`, + runtime_id: lane.name, + label: laneLabel(lane, byId, runId), + devices, + device_count: devices.length, + start: held.length ? Math.min(...held.map(([a]) => a)) : 0, + end: held.length ? Math.max(...held.map(([, b]) => b)) : 0, + intervals, + waits, + ops: active.length ? { [opName]: active } : {}, + exact, + }; + }); +} + +export const dutyBands = (lanes, device = null) => + lanes + .filter((p) => p.intervals.length && (!device || p.devices.includes(device))) + .map((p) => ({ key: p.id, label: `${p.label} · ${p.role || "process"}`, hue: placementColor(p), role: p.role, spans: p.intervals })); + +export const deviceLabel = (name) => /(\d+)$/.exec(String(name ?? ""))?.[1] ?? String(name ?? ""); +export const acceleratorLabel = (node) => + node.accelerator ? node.accelerator.replace(/^nvidia[- ]/i, "").replace(/-\d+gb$/i, "").replace(/-/g, " ").toUpperCase() : node.gpu_capacity ? "GPU" : "CPU"; + +export function clipPlacements(placements = [], range, error = null, status = null) { + const clip = (spans = []) => spans.map(([a, b]) => [Math.max(range.start, a), Math.min(range.now, b)]).filter(([a, b]) => b > a); + return placements.map((p) => ({ + ...p, + intervals: clip(p.intervals), + waits: clip(p.waits), + ops: Object.fromEntries(Object.entries(p.ops || {}).map(([name, spans]) => [name, clip(spans)])), + error: status ? error : p.error, + errorStatus: status || p.errorStatus, + })); +} + +export function activitySource(placements) { + if (!placements.length) return ""; + const hasOps = placements.some((p) => Object.keys(p.ops || {}).length); + const source = placements.every((p) => p.exact) + ? "GPU turns" + : placements.some((p) => p.exact) + ? "GPU turns and recorded operations" + : hasOps + ? "Recorded operations; may include GPU wait time" + : "Workload GPU placements"; + return `${source}. Gaps may include unrecorded activity.`; +} + +export function processLabel(p, nodes, { gpus: named = true } = {}) { + const node = nodes.find((n) => n.name === p.node), devices = p.devices.map((id) => node?.devices.find((d) => d.id === id)); + const gpus = devices.length && devices.every(Boolean) + ? `GPU${devices.length === 1 ? "" : "s"} ${devices.map((d) => deviceLabel(d.name)).join(", ")}` + : p.device_count ? `${p.device_count} GPU${p.device_count === 1 ? "" : "s"}` : null; + const role = p.role || "Process", accelerator = node && acceleratorLabel(node); + return [role[0].toUpperCase() + role.slice(1), accelerator === "GPU" ? null : accelerator, named && gpus].filter(Boolean).join(" · "); +} + +export const OP_ORDER = ["training", "forward_backward", "forward", "optim_step", "sample", "init", "wake_up", "sleep", "weight_sync"]; +export const opRank = (name) => OP_ORDER.includes(name) ? OP_ORDER.indexOf(name) : OP_ORDER.length; +export const opLabel = (name) => name.replace(/_/g, " "); + +export function claimLabel(node, placements, live) { + if (live && !node.ready) return "Offline"; + if (!node.gpu_capacity) return "CPU"; + if (node.openrl_enabled === false) return "Not enabled"; + if (!placements.every((p) => p.devices.length === p.device_count && p.devices.every((id) => node.devices.some((d) => d.id === id)))) return "— claimed"; + return `${new Set(placements.flatMap((p) => p.devices)).size}/${node.gpu_capacity} claimed`; +} + +export function capacityBars(devices, placements, range, selected) { + const height = devices.length === 1 ? 30 : 22; + const span = (from, to) => + `left:${100 * Math.max(0, Math.max(range.start, from) - range.start) / (range.now - range.start)}%;width:${100 * Math.max(0, Math.min(range.now, to) - Math.max(range.start, from)) / (range.now - range.start)}%`; + return devices.flatMap((device, i) => { + const rows = placements.filter((p) => p.devices.includes(device.id)).sort((a, b) => a.start - b.start || a.id.localeCompare(b.id)); + if (!rows.length) return []; + const from = Math.max(range.start, Math.min(...rows.map((p) => p.start))), to = Math.min(range.now, Math.max(...rows.map((p) => p.end))); + return [{ + key: `gpu:${device.id}`, + placement: rows.find((p) => p.id === selected) || rows.at(-1), + selected: rows.some((p) => p.id === selected), + title: `GPU activity: ${rows.map((p) => `${p.label} ${p.role || ""}`.trim()).join(", ")}`, + style: `${span(from, to)};top:${i * height + 2}px;height:${height - 4}px`, + holds: rows.flatMap((p) => + p.intervals.map(([a, b]) => ({ + placement: p, + style: `left:${100 * (Math.max(a, from) - from) / (to - from || 1)}%;width:${100 * (Math.min(b, to) - Math.max(a, from)) / (to - from || 1)}%`, + }))), + }]; + }); +} + +export function gapFor(sortedPoints, floor) { + const spacings = sortedPoints.slice(1).map(([at], i) => at - sortedPoints[i][0]).filter((d) => d > 0).sort((a, b) => a - b); + return Math.max(floor, 3 * (spacings[Math.floor(spacings.length / 2)] || 0)); +} + +export function deviceSeries(devices, field) { + const slots = new Set(devices.map((d) => d.id)).size, byTime = new Map(); + for (const device of devices) { + for (const [at, value] of device[field] || []) { + if (!byTime.has(at)) byTime.set(at, new Map()); + const values = byTime.get(at), samples = values.get(device.id) || []; + if (Number.isFinite(value)) samples.push(value); + values.set(device.id, samples); + } + } + return [...byTime] + .sort((a, b) => a[0] - b[0]) + .map(([at, values]) => [ + at, + values.size === slots && [...values.values()].every((samples) => samples.length === 1) + ? [...values.values()].reduce((sum, samples) => sum + samples[0], 0) / (field === "utilization" ? slots : 1) + : null, + ]); +} + +export const lastValue = (points, range) => points.findLast(([at]) => at >= range.start && at <= range.now)?.[1]; diff --git a/tests/test_observability.py b/tests/test_observability.py index 859a7c67a..1b02c9607 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -347,5 +347,23 @@ def test_a_run_without_a_client_log_id_has_no_client_logs(self): self.assertEqual(self.sent, []) +class DashboardStaticTest(unittest.TestCase): + def test_dashboard_serves_index_and_static_assets_and_rejects_traversal(self): + client = TestClient(api_server.app) + for path in ("/dashboard", "/dashboard/"): + response = client.get(path) + self.assertEqual(response.status_code, 200) + self.assertIn("/dashboard/assets/app.js", response.text) + self.assertEqual(response.headers.get("cache-control"), "no-store") + css = client.get("/dashboard/assets/style.css") + self.assertEqual(css.status_code, 200) + self.assertIn("text/css", css.headers.get("content-type", "")) + js = client.get("/dashboard/assets/app.js") + self.assertEqual(js.status_code, 200) + self.assertIn("javascript", js.headers.get("content-type", "")) + self.assertEqual(client.get("/dashboard/assets/missing.js").status_code, 404) + self.assertEqual(client.get("/dashboard/assets/..%2Frouter.py").status_code, 404) + + if __name__ == "__main__": unittest.main()