From 3445764826cb55c971b30ae119b7328e7ce9d2da Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:20:05 -0500 Subject: [PATCH 1/2] fix: preserve Klaud baseline preflight and classify failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在候选任务中复用规划阶段的基线预检，并记录脱敏失败原因。 --- .github/klaud-candidate-prompt.md | 19 ++++- .github/workflows/klaud-plan.yml | 1 + KLAUD_DEBUG.md | 18 ++++ docs/klaud-reporting.md | 2 +- docs/klaud-reporting_zh.md | 2 +- docs/klaud.md | 6 +- docs/klaud_zh.md | 6 +- infx/klaud/__main__.py | 34 ++++++-- infx/klaud/lifecycle.py | 12 +-- infx/klaud/models.py | 13 +++ infx/klaud/reporting.py | 33 ++++++- utils/test_klaud_github.py | 137 +++++++++++++++++++++++++++++- 12 files changed, 256 insertions(+), 27 deletions(-) diff --git a/.github/klaud-candidate-prompt.md b/.github/klaud-candidate-prompt.md index 21161a5a8..6d996fd3a 100644 --- a/.github/klaud-candidate-prompt.md +++ b/.github/klaud-candidate-prompt.md @@ -58,9 +58,12 @@ Freeze the COMPLETE original public baseline point roster before attempts, using candidate.source.date, verified old-image producer IDs/SHAs and full recipe/workload/topology/ concurrency/dataset identities. Use the reporting guide's prepare-baseline/report commands; candidate.json provides the planner-verified exact `baseline-model`; use that value unchanged. -The planner's preflight proves that a full roster can be reconstructed but does not freeze or -publish it. Prepare the candidate's own baseline after resolving the exact old/new image goal; -the helper recovers original points from producer revisions. Supplement verified public +The planner supplies `baseline-preflight.json` beside candidate.json. It contains the verified +benchmark roster bound to the selected candidate, base SHA, source observation and model. +After resolving the exact old/new image goal, prepare-baseline checks this binding and uses +that roster without refetching it. If candidate.json requires the preflight and it is absent +or invalid, stop with `baseline-preflight-mismatch`; only legacy candidates may reconstruct. +The preflight is not a published or final baseline. Supplement verified public eval/dataset evidence before freezing; never replace a failed lookup with a partial roster. Never reduce the baseline to overlapping points, displayed rows or a smaller current family. Never dispatch the old image. Unproven deltas are N/A with a reason; N/A never excuses missing updated-image results. @@ -123,6 +126,16 @@ On terminal resolve/baseline evidence, stop investigating. Finalize attempts, wr CandidateOutcome to $KLAUD_EVIDENCE/requested-outcome.json and run `finish --outcome-file "$KLAUD_EVIDENCE/requested-outcome.json"`. Only finish may mark ready after verifying complete artifacts and publishing the final report BEFORE reviews begin. +For `failed` in phase `baseline`, include one fixed `reason-code` in CandidateOutcome: +`baseline-preflight-mismatch` (candidate/base/source/model binding differs), +`baseline-provenance-unverified` (old-image point producer or SHA cannot be proven), +`baseline-point-mismatch` (roster identity, topology or coverage is ambiguous), +`baseline-eval-unverified` (required published eval or dataset evidence cannot be proven), +`baseline-api-unavailable` (public evidence read failed), or `baseline-other` (none of these +can be established). Select the first actual blocker, not a guessed cause. A failure before +baseline work belongs in `resolve`, and a failed updated-image run belongs in `targeted` or +`final-sweep`. Do not put free text, raw responses, private telemetry or transcripts in the +code. The fixed code appears in sanitized diagnostics and the completion report. After finish returns a verified `validated` outcome, check for an existing exact `/use ` comment, then post it once on this PR. Never post `/use` for any other outcome or any run except the verified final sweep. diff --git a/.github/workflows/klaud-plan.yml b/.github/workflows/klaud-plan.yml index 5826854e5..7fce5f021 100644 --- a/.github/workflows/klaud-plan.yml +++ b/.github/workflows/klaud-plan.yml @@ -163,6 +163,7 @@ jobs: ${{ runner.temp }}/klaud/selection.json ${{ runner.temp }}/klaud/review-diagnostics.json ${{ runner.temp }}/klaud/*/candidate.json + ${{ runner.temp }}/klaud/*/baseline-preflight.json retention-days: 14 candidates: diff --git a/KLAUD_DEBUG.md b/KLAUD_DEBUG.md index e24a5832c..76be01cb9 100644 --- a/KLAUD_DEBUG.md +++ b/KLAUD_DEBUG.md @@ -402,3 +402,21 @@ are skipped, and registry `/` and enroot `#` image spellings are normalized for historical identity matching and point backfill. A transient or malformed baseline for one candidate defers that candidate and continues through the reviewed pool; it does not consume or block later candidate slots. + +### 7.8 Selected candidates can still stop in the baseline phase + +**Symptom:** a parent workflow and candidate wrapper jobs succeed, but sanitized +candidate outcomes say `failed` / `baseline`, with no validation runs. A draft PR +may have been created and then closed. The wrapper status only means the agent +reported and cleaned up; it does not mean the image update worked. + +**Diagnosis:** planner preflight already reconstructed the public benchmark roster. +The selected candidate now receives that typed roster in `baseline-preflight.json`. +`prepare-baseline` verifies its candidate/base/source/model binding and reuses it, +then the agent still verifies additional eval/dataset evidence before publication. +Read the fixed `reason-code` in `candidate-diagnostics.json`, the job summary or +the completion receipt to identify the first verified blocker. An absent code on +an older run means the exact cause was not recorded; do not infer one from the phase. +Never publish raw API responses, agent transcripts or private telemetry to fill +that gap. A preflight mismatch must stop rather than silently refetching another +roster or shrinking coverage. diff --git a/docs/klaud-reporting.md b/docs/klaud-reporting.md index 4682bf6ef..62d859e7b 100644 --- a/docs/klaud-reporting.md +++ b/docs/klaud-reporting.md @@ -29,7 +29,7 @@ KLAUD=(uv run --no-project --exclude-newer PT12H --python 3.12 \ "${KLAUD[@]}" report --kind attempt --file "$KLAUD_EVIDENCE/attempt.json" ``` -`prepare-baseline` queries public `benchmarks` with the selected date, `exact=true`, and no calculator view; `workflow-info` establishes producer IDs, heads and attempts. It reconstructs the selected family from each producer's YAML with trusted local generator code, including the historical `.github/configs` layout and flat runner-label format. Only that family is validated, so retired sibling schemas cannot break reconstruction. Matching requires the old image and full public workload/topology/concurrency identity; supplied recipe fingerprints must also match. Legacy rows without fingerprints require a unique match and a producer changelog selecting the family. Rows outside the reconstructed family are ignored even when their producer metadata is incomplete. Missing provenance on a matching current or historical point, ambiguous identities and duplicate points stop preparation instead of publishing an incomplete baseline. Before the first publication, supplement verified published evals and dataset provenance using [the public API routes](./klaud.md#public-api-investigation). `BenchmarkRow` has no dataset identity; unproven AgentX datasets still produce N/A deltas. Never run the old image to fill a gap. +`prepare-baseline` verifies the candidate/base/source/model binding in the planner-produced `baseline-preflight.json` and reuses its public benchmark roster. A selected candidate marked `baseline-preflight-required` stops if this file is missing or invalid. A legacy candidate without this file queries public `benchmarks` with the selected date, `exact=true`, and no calculator view; `workflow-info` establishes producer IDs, heads and attempts. It reconstructs the selected family from each producer's YAML with trusted local generator code, including the historical `.github/configs` layout and flat runner-label format. Only that family is validated, so retired sibling schemas cannot break reconstruction. Matching requires the old image and full public workload/topology/concurrency identity; supplied recipe fingerprints must also match. Legacy rows without fingerprints require a unique match and a producer changelog selecting the family. Rows outside the reconstructed family are ignored even when their producer metadata is incomplete. Missing provenance on a matching current or historical point, ambiguous identities and duplicate points stop preparation instead of publishing an incomplete baseline. Before the first publication, supplement verified published evals and dataset provenance using [the public API routes](./klaud.md#public-api-investigation). `BenchmarkRow` has no dataset identity; unproven AgentX datasets still produce N/A deltas. Never run the old image to fill a gap. The baseline file is created once; retries do not refetch it. The PR body freezes the complete typed record in a compressed hidden marker and renders the complete baseline once. Conflicting replacement records are rejected. Existing PRs with the older baseline-comment format remain readable for recovery. A correction requires a maintainer to review the evidence and make the correction explicit; do not silently revise the baseline during repairs. diff --git a/docs/klaud-reporting_zh.md b/docs/klaud-reporting_zh.md index e92ac029b..c0153590b 100644 --- a/docs/klaud-reporting_zh.md +++ b/docs/klaud-reporting_zh.md @@ -29,7 +29,7 @@ KLAUD=(uv run --no-project --exclude-newer PT12H --python 3.12 \ "${KLAUD[@]}" report --kind attempt --file "$KLAUD_EVIDENCE/attempt.json" ``` -`prepare-baseline` 使用候选日期、`exact=true` 查询公开 `benchmarks`，不使用 calculator view；通过 `workflow-info` 核实产出运行 ID、head 和运行次数。它用受信任的本地生成器，从各产出提交的 YAML 重建所选配置族，支持历史 `.github/configs` 路径和平铺 runner 标签格式。只校验所选配置族，避免已退役兄弟配置的旧 schema 阻断重建。匹配要求旧镜像及完整公开工作负载、拓扑、并发身份一致；存在 recipe fingerprint 时也必须匹配。没有指纹的旧数据必须唯一匹配，且产出运行的 changelog 必须选择该配置族。不属于重建配置族的数据行会被忽略，即使其产出运行元数据不完整。匹配当前或历史测试点的数据行若来源缺失，或存在身份歧义、重复点，会阻止准备，不发布不完整基线。首次发布前，按[公开 API 路由](./klaud_zh.md#公开-api-调查)补充已验证的历史评测和数据集来源。`BenchmarkRow` 本身不含数据集身份；AgentX 数据集无法核实时，差值仍为 N/A。绝不通过运行旧镜像补齐基线。 +`prepare-baseline` 核对规划阶段 `baseline-preflight.json` 中候选、基准 SHA、来源和模型的绑定，并复用其公开 benchmark 测试点清单。标记 `baseline-preflight-required` 的所选候选在文件缺失或无效时停止。没有该文件的旧候选使用候选日期、`exact=true` 查询公开 `benchmarks`，不使用 calculator view；通过 `workflow-info` 核实产出运行 ID、head 和运行次数。它用受信任的本地生成器，从各产出提交的 YAML 重建所选配置族，支持历史 `.github/configs` 路径和平铺 runner 标签格式。只校验所选配置族，避免已退役兄弟配置的旧 schema 阻断重建。匹配要求旧镜像及完整公开工作负载、拓扑、并发身份一致；存在 recipe fingerprint 时也必须匹配。没有指纹的旧数据必须唯一匹配，且产出运行的 changelog 必须选择该配置族。不属于重建配置族的数据行会被忽略，即使其产出运行元数据不完整。匹配当前或历史测试点的数据行若来源缺失，或存在身份歧义、重复点，会阻止准备，不发布不完整基线。首次发布前，按[公开 API 路由](./klaud_zh.md#公开-api-调查)补充已验证的历史评测和数据集来源。`BenchmarkRow` 本身不含数据集身份；AgentX 数据集无法核实时，差值仍为 N/A。绝不通过运行旧镜像补齐基线。 基线文件仅创建一次，重试不重新获取。PR 正文通过压缩的隐藏标记冻结完整类型化记录，并只渲染一次完整基线；冲突替换会被拒绝。旧 PR 使用的基线评论格式仍可供恢复流程读取。确需修正时，由维护者检查证据并明确记录更正，不能在修复期间静默改变基线。 diff --git a/docs/klaud.md b/docs/klaud.md index af96564ea..8190e38ec 100644 --- a/docs/klaud.md +++ b/docs/klaud.md @@ -25,7 +25,7 @@ Both agents receive explicit evidence-directory access. The overlap reviewer use The private gate requires **node utilization strictly below 80%**: `(summary.allocatedNodes + summary.mixedNodes) * 5 < summary.totalNodes * 4`, without rounding. Both fully allocated and partially used nodes count as in use; exactly 80% is rejected. There is no reserved-node deduction. At least one idle node is required so an entirely unavailable cluster cannot pass as 0% utilized. Missing, invalid, inconsistent, stale or unavailable data is rejected. Compatibility is determined from the consumed fields, not `schemaVersion`; additive fields and version changes do not exclude otherwise valid clusters. Hardware matching is only the initial shortlist; the review must resolve every actual target, and selection rechecks those exact IDs. Eligible jobs may queue while the scheduler waits for their complete physical node demand to fit. -The `klaud-plan` artifact explicitly includes only `candidates.json`, `open-prs.json`, `selection.json`, `review-diagnostics.json` and each selected candidate's `candidate.json`. The local `capacity.json` contains telemetry IDs and eligibility hints for review and is **never uploaded**, nor are arbitrary scratch files. Each handoff contains the published baseline observation, verified `baseline-model`, branch, base SHA, public benchmark discovery URLs and validated `pr-review`, without private node counts or raw telemetry. Selected candidates run in parallel, each with its own Klaud Cold session. A failed candidate does not cancel the others. There is no copied model/runner catalog or `recipes.py`; live recipe and upstream image interpretation belong to the agent using existing InferenceX configuration and tooling. +The `klaud-plan` artifact explicitly includes only `candidates.json`, `open-prs.json`, `selection.json`, `review-diagnostics.json` and each selected candidate's `candidate.json` and `baseline-preflight.json`. The local `capacity.json` contains telemetry IDs and eligibility hints for review and is **never uploaded**, nor are arbitrary scratch files. Each handoff contains the published baseline observation, verified `baseline-model`, branch, base SHA, public benchmark discovery URLs and validated `pr-review`, without private node counts or raw telemetry. Selected candidates run in parallel, each with its own Klaud Cold session. A failed candidate does not cancel the others. There is no copied model/runner catalog or `recipes.py`; live recipe and upstream image interpretation belong to the agent using existing InferenceX configuration and tooling. ## Klaud Cold owns execution @@ -57,13 +57,13 @@ Klaud Cold dispatches `e2e-tests.yml` with the explicit boolean input `klaud-run ### Published baseline and session completion -The baseline comes from the **public dashboard API at `https://inferencex.semianalysis.com`**. Planning resolves the OpenAPI display model and preflights the full roster before agent dispatch. The candidate uses that exact `candidate.json` value with `candidate.source.date`, `workflow-info` and `benchmarks`, sets `date` and `exact=true`, and omits the calculator `view`. The preflight is an eligibility check, so the candidate still freezes and publishes its own baseline after resolving the exact old/new image goal. Verify the old image and full model/hardware/framework/precision/speculative-decoding/workload identity, then match topology, concurrency and dataset per point. Record API queries, published date and each point's producer `run_url`/SHA, keeping logical curve snapshots distinct from producers. Retrieve published evals when relevant and freeze this baseline across attempts. Missing or incomparable data means `N/A` with a reason. Never dispatch or rerun an old-image baseline. +The baseline comes from the **public dashboard API at `https://inferencex.semianalysis.com`**. Planning resolves the OpenAPI display model and preflights the full roster before agent dispatch. The candidate uses that exact `candidate.json` value with `candidate.source.date`, `workflow-info` and `benchmarks`, sets `date` and `exact=true`, and omits the calculator `view`. The planner saves its verified benchmark roster in `baseline-preflight.json`, bound to the candidate ID, base SHA, source observation and display model. After resolving the exact old/new image goal, `prepare-baseline` verifies that binding and uses the same roster without a second benchmark API fetch; a selected candidate marked `baseline-preflight-required` stops if the artifact is missing or invalid; only legacy candidates reconstruct it. The candidate still supplements public eval/dataset evidence and freezes and publishes its own final baseline. Verify the old image and full model/hardware/framework/precision/speculative-decoding/workload identity, then match topology, concurrency and dataset per point. Record API queries, published date and each point's producer `run_url`/SHA, keeping logical curve snapshots distinct from producers. Retrieve published evals when relevant and freeze this baseline across attempts. Missing or incomparable data means `N/A` with a reason. Never dispatch or rerun an old-image baseline. Dispatching a run or opening a draft is not completion. Stay in the same session with `gh run watch --interval 60`, resuming after tool timeouts, and inspect job-level state because queued workflows can contain running jobs. A failed benchmark matrix may leave eval jobs active. Diagnose the first server error rather than teardown symptoms; repair within the existing scope, budget and capacity rule. A tool denial calls for an allowed tool or command, not early success. Finalize the PR attempt comments with terminal results, then report the stop reason, repair count, confirmed child completion and PR URL. Never end with a promise to monitor later or cancel healthy work just to stop. The [Stop hook](https://code.claude.com/docs/en/hooks#stop) runs `check-stop` against `$KLAUD_EVIDENCE/outcome.json`. Before ending, write the requested `CandidateOutcome` JSON to a separate file and run `uv run --no-project --exclude-newer PT12H --python 3.12 --with "pydantic>=2.10,<3" --with pyyaml python -m infx.klaud finish --outcome-file "$KLAUD_EVIDENCE/requested-outcome.json"`. Then invoke `StructuredOutput` exactly once with the verified `outcome.json` object, without prose or encoding it as a string. Diagnostics prefer this verified file over the action's duplicate structured-output copy and fall back to the copy only when the file is unavailable or invalid. The command discovers all owned targeted/final runs from the original parent creation time, including earlier heads and closed or unlabeled PRs. A failure/deferral report precedes cancellation; closure waits for terminal jobs, removes sweep labels, returns to draft and deletes the unchanged exact-head branch for every unsuccessful outcome. The completion receipt records actual run IDs and cleanup. Pending cancellation or PR-transition jobs require waiting and retrying `finish`. A maintainer handoff supersedes cleanup. Unrelated owners, forks, moving heads, merged PRs and ambiguous state are never overwritten. Without an owned PR, unverified branches are left alone. The hook only verifies; it does not mutate or overcome Claude’s built-in stop-loop cap. -After every completed agent step, regardless of its action outcome, `recover-current` makes one trusted nonblocking reconciliation pass. This immediately releases a no-PR/no-run session, preserves healthy child work for later recovery, or completes terminal cleanup/validation. Diagnostics then prefer a GitHub-verified lifecycle receipt even if the SDK omitted structured output. Fixed error codes distinguish unavailable session state, an invalid receipt, invalid structured output and an outcome that fails lifecycle verification. The sanitized artifact is uploaded before an unverifiable outcome fails the candidate job. Only fixed outcome categories, numeric IDs/metrics and head/attempt references are retained; never raw execution messages, commands, credentials or private responses. +After every completed agent step, regardless of its action outcome, `recover-current` makes one trusted nonblocking reconciliation pass. This immediately releases a no-PR/no-run session, preserves healthy child work for later recovery, or completes terminal cleanup/validation. Diagnostics then prefer a GitHub-verified lifecycle receipt even if the SDK omitted structured output. Fixed error codes distinguish unavailable session state, an invalid receipt, invalid structured output and an outcome that fails lifecycle verification. The sanitized artifact is uploaded before an unverifiable outcome fails the candidate job. A failed baseline outcome must carry a fixed `reason-code` for the first verified blocker: preflight binding, producer provenance, point identity, eval evidence, public API, or other. The verified receipt, sanitized candidate artifact, job summary and PR completion comment retain the code; historical receipts without one remain readable. Only fixed outcome categories, numeric IDs/metrics and head/attempt references are retained; never raw execution messages, commands, credentials or private responses. `run-sweep.yml` uploads `klaud-sweep-manifest` with the full matrix, exact head and run ID/attempt. `check-final` and final validation independently generate the unfiltered family from exact-head YAML using trusted code. Equivalent scenario filters can pass; missing/changed points and default evals cannot. Verification selects the current-attempt manifest and newest same-name results from that same run/head. Earlier aggregates survive only if their point producers did not rerun; all benchmark/eval coverage and raw/aggregate consistency still must pass. Archives are never overlaid. Missing/expired artifacts and generator-policy drift require inspection. Older runs without a manifest cannot be certified automatically. diff --git a/docs/klaud_zh.md b/docs/klaud_zh.md index b1bbd8307..1e5a8a3d1 100644 --- a/docs/klaud_zh.md +++ b/docs/klaud_zh.md @@ -25,7 +25,7 @@ PR 检查使用 `claude-opus-5`（Opus 5），关闭 fast mode（`fastMode: fals 私有容量门槛是 **节点利用率严格低于 80%**：`(summary.allocatedNodes + summary.mixedNodes) * 5 < summary.totalNodes * 4`，不做舍入。完全分配和部分使用的节点均计入已使用节点；恰好 80% 时不放行。不扣除预留节点。要求至少有一个空闲节点，避免整个集群不可用时仍以 0% 利用率通过检查。缺失、无效、不一致、过期或不可用的数据均拒绝。硬件匹配只用于初筛；检查阶段必须解析每个实际目标，选择阶段重新检查这些精确 ID。符合条件的作业可以先排队，由调度器等待完整物理节点需求能够满足后再启动。兼容性按实际读取的字段判断，不依赖 `schemaVersion`；新增字段或版本变化不会排除其他方面均有效的集群。 -`klaud-plan` 产物仅显式包含 `candidates.json`、`open-prs.json`、`selection.json`、`review-diagnostics.json` 及每个所选候选的 `candidate.json`。本地 `capacity.json` 为检查阶段提供遥测 ID 和资格线索，**绝不上传**；任意临时文件也不会上传。每份交接文件包含已发布基线观测、已验证的 `baseline-model`、分支、基准 SHA、公开 benchmark 查询 URL 和通过校验的 `pr-review`，不包含私有节点计数或原始遥测。所选候选并行运行，各自获得独立 Klaud Cold 会话。一个候选失败不会取消其他候选。不再复制模型/runner 目录，也不保留 `recipes.py`；agent 使用现有 InferenceX 配置和工具理解实际 recipe 及上游镜像。 +`klaud-plan` 产物仅显式包含 `candidates.json`、`open-prs.json`、`selection.json`、`review-diagnostics.json` 及每个所选候选的 `candidate.json` 和 `baseline-preflight.json`。本地 `capacity.json` 为检查阶段提供遥测 ID 和资格线索，**绝不上传**；任意临时文件也不会上传。每份交接文件包含已发布基线观测、已验证的 `baseline-model`、分支、基准 SHA、公开 benchmark 查询 URL 和通过校验的 `pr-review`，不包含私有节点计数或原始遥测。所选候选并行运行，各自获得独立 Klaud Cold 会话。一个候选失败不会取消其他候选。不再复制模型/runner 目录，也不保留 `recipes.py`；agent 使用现有 InferenceX 配置和工具理解实际 recipe 及上游镜像。 ## Klaud Cold 负责执行 @@ -57,13 +57,13 @@ Klaud Cold 调度 `e2e-tests.yml` 时显式设置布尔输入 `klaud-run: true` ### 已发布基线与会话完成 -基线来自 **`https://inferencex.semianalysis.com` 的公开 dashboard API**。规划阶段先解析 OpenAPI 展示模型名，并在启动 agent 前预检完整测试点清单。候选使用 `candidate.json` 中这一精确值，将 `candidate.source.date` 传给 `workflow-info` 和 `benchmarks`，设置 `date` 和 `exact=true`，不使用计算器 `view`。预检只判断资格；候选解析精确的新旧镜像目标后，仍须冻结并发布自己的基线。核实旧镜像以及完整的模型、硬件、框架、精度、推测解码和工作负载身份，再逐点匹配拓扑、并发量及数据集。记录 API 查询、发布日期和每个测试点的来源 `run_url`/SHA，区分逻辑曲线快照与实际数据来源。按需读取已发布 eval，所有尝试共用这份固定基线。缺失或不可比较的数据填写 `N/A` 并说明原因。绝不调度或重跑旧镜像基线。 +基线来自 **`https://inferencex.semianalysis.com` 的公开 dashboard API**。规划阶段先解析 OpenAPI 展示模型名，并在启动 agent 前预检完整测试点清单。候选使用 `candidate.json` 中这一精确值，将 `candidate.source.date` 传给 `workflow-info` 和 `benchmarks`，设置 `date` 和 `exact=true`，不使用计算器 `view`。规划阶段将已验证的 benchmark 测试点清单写入 `baseline-preflight.json`，并绑定候选 ID、基准 SHA、来源观测和展示模型。候选解析精确的新旧镜像目标后，`prepare-baseline` 核对这些绑定并复用同一清单，不再重复读取 benchmark API；标记 `baseline-preflight-required` 的所选候选在产物缺失或无效时停止；仅旧候选按原路径重建。候选仍须补充公开 eval／数据集证据，并冻结、发布最终基线。核实旧镜像以及完整的模型、硬件、框架、精度、推测解码和工作负载身份，再逐点匹配拓扑、并发量及数据集。记录 API 查询、发布日期和每个测试点的来源 `run_url`/SHA，区分逻辑曲线快照与实际数据来源。按需读取已发布 eval，所有尝试共用这份固定基线。缺失或不可比较的数据填写 `N/A` 并说明原因。绝不调度或重跑旧镜像基线。 调度运行或创建草稿不代表任务完成。使用 `gh run watch --interval 60` 留在同一会话中等待，工具超时后继续等待，并检查作业级状态，因为 queued 工作流可能包含正在运行的作业。benchmark 矩阵失败后，eval 作业仍可能继续。定位首个服务端错误而非清理阶段症状；在原有范围、预算和容量规则内修复。工具调用被拒绝时改用允许的工具或命令，不得提前报告成功。先将所有尝试的最终结果写入 PR 尝试评论，再报告停止原因、修复次数、已确认的子运行结束状态和 PR URL。不得承诺稍后继续监控，也不得仅为结束会话而取消正常运行。 [Stop hook](https://code.claude.com/docs/en/hooks#stop) 通过 `check-stop` 检查 `$KLAUD_EVIDENCE/outcome.json`。结束前，将请求的 `CandidateOutcome` JSON 写入单独文件，运行 `uv run --no-project --exclude-newer PT12H --python 3.12 --with "pydantic>=2.10,<3" --with pyyaml python -m infx.klaud finish --outcome-file "$KLAUD_EVIDENCE/requested-outcome.json"`，然后仅调用一次 `StructuredOutput`，传入已验证的 `outcome.json` 对象，不附加说明文字，也不将其编码为字符串。诊断优先使用该验证文件；仅在文件不可用或无效时，才回退到 action 的重复结构化输出。命令从父运行原始创建时间起发现所有自有定向和最终运行，包括旧 head、已关闭或移除标签的 PR。先发布失败或延后报告，再取消运行；全部结束后才移除 sweep 标签、退回草稿、关闭 PR，并为所有失败结果删除未移动的精确 head 分支。完成记录包含实际运行 ID 和清理状态。取消或 PR 状态转换事件的作业仍在进行时，等待后重试 `finish`。维护者接管优先于清理；不覆盖其他所有者、fork、已移动的 head、已合并 PR 或不明确的状态。没有所属 PR 时不删除无法验证归属的分支。hook 仅做验证，不修改状态，也不能突破 Claude 内置的停止循环上限。 -每次 agent 步骤结束后，无论 action 结果如何，`recover-current` 都执行一次受信任的非阻塞收尾。没有 PR/run 的会话会立即释放；健康子运行保留给后续恢复；已结束工作则完成清理或验证。随后诊断优先使用经 GitHub 验证的生命周期记录。固定错误码区分会话状态不可用、记录无效、结构化输出无效和生命周期验证失败。无法验证时，先上传脱敏产物，再让候选作业失败。仅记录固定结果类别、数字 ID/指标及 head/attempt，不包含原始执行消息、命令、凭据或私有响应。 +每次 agent 步骤结束后，无论 action 结果如何，`recover-current` 都执行一次受信任的非阻塞收尾。没有 PR/run 的会话会立即释放；健康子运行保留给后续恢复；已结束工作则完成清理或验证。随后诊断优先使用经 GitHub 验证的生命周期记录。固定错误码区分会话状态不可用、记录无效、结构化输出无效和生命周期验证失败。无法验证时，先上传脱敏产物，再让候选作业失败。基线阶段的失败结果必须附带固定 `reason-code`，标明首个经核实的阻碍：预检绑定、来源溯源、测试点身份、eval 证据、公开 API 或其他。经验证的完成记录、脱敏候选产物、作业摘要和 PR 完成评论保留该代码；没有代码的历史记录仍可读取。仅记录固定结果类别、数字 ID/指标及 head/attempt，不包含原始执行消息、命令、凭据或私有响应。 `run-sweep.yml` 上传包含完整矩阵、精确 head 和 run ID/attempt 的 `klaud-sweep-manifest`。`check-final` 与最终验证均使用受信任代码，从精确 head 的 YAML 独立生成未过滤配置族。覆盖等价的 scenario filter 可以通过，缺失或改变的配置点与默认 eval 不能通过。验证器选择当前 attempt 的 manifest，以及同一 run/head 下每个名称最新的产物。仅当对应配置生产作业没有重跑时，才保留之前的 aggregate；全部覆盖及原始结果/汇总一致性仍须通过。不同 archive 不叠加解压。缺失、过期产物及生成器策略变化需要检查；无 manifest 的旧运行不能自动认证。 diff --git a/infx/klaud/__main__.py b/infx/klaud/__main__.py index a7e1092ae..ef820770b 100644 --- a/infx/klaud/__main__.py +++ b/infx/klaud/__main__.py @@ -510,7 +510,7 @@ def save_diagnostics( diagnostics["outcome-report"] = "unavailable-or-invalid" else: diagnostics["outcome-report"] = "available" - diagnostics["candidate-outcome"] = outcome.model_dump(by_alias=True) + diagnostics["candidate-outcome"] = outcome.model_dump(by_alias=True, exclude_unset=True) output.parent.mkdir(parents=True, exist_ok=True) output.write_text(json.dumps(diagnostics, indent=2) + "\n") if filename := os.environ.get("GITHUB_STEP_SUMMARY"): @@ -525,18 +525,19 @@ def save_diagnostics( repairs = ( str(outcome.repairs_used) if outcome.repairs_used is not None else "unknown / 未知" ) + reason = outcome.reason_code or "—" with open(filename, "a") as summary: summary.write( - "| Outcome / 结果 | Phase / 阶段 | PR | Repairs / 修复 | Runs / 运行 |\n" - "| --- | --- | --- | --- | --- |\n" - f"| {outcome.outcome} | {outcome.phase} | {pr} | {repairs} | {runs} |\n" + "| Outcome / 结果 | Phase / 阶段 | Reason / 原因 | PR | Repairs / 修复 | Runs / 运行 |\n" + "| --- | --- | --- | --- | --- | --- |\n" + f"| {outcome.outcome} | {outcome.phase} | {reason} | {pr} | {repairs} | {runs} |\n" ) return outcome.outcome != "unexpected-error" def select(directory: Path, max_candidates: int, execution_file: Path | None = None) -> None: from . import claims - from .reporting import Prose, resolve_baseline + from .reporting import BaselinePreflight, Prose, resolve_baseline contexts = json.loads((directory / "candidates.json").read_text()) review = PRReview(decisions=[]) @@ -572,6 +573,7 @@ def select(directory: Path, max_candidates: int, execution_file: Path | None = N deferred = "capacity-unavailable" capacity_deferred = [] baseline_deferred = [] + preflights = {} families = {decision.family for decision in review.decisions if decision.decision != "proceed"} for candidate in contexts: decision = decisions.get(candidate["id"]) @@ -584,7 +586,7 @@ def select(directory: Path, max_candidates: int, execution_file: Path | None = N {key: candidate[key] for key in ("id", "family", "base")} ) try: - resolve_baseline( + baseline = resolve_baseline( os.environ["GITHUB_REPOSITORY"], owned, candidate, @@ -594,6 +596,13 @@ def select(directory: Path, max_candidates: int, execution_file: Path | None = N zh="在调度候选任务前验证完整的已发布基线。", ), ) + preflight = BaselinePreflight( + candidate_id=candidate["id"], + base=candidate["base"], + baseline_model=decision.baseline_model, + source_identity=identity(candidate["source"]), + baseline=baseline, + ) except VerificationError: baseline_deferred.append(candidate["id"]) families.add(decision.family) @@ -610,9 +619,11 @@ def select(directory: Path, max_candidates: int, execution_file: Path | None = N { **candidate, "baseline-model": decision.baseline_model, + "baseline-preflight-required": True, "pr-review": decision.model_dump(by_alias=True), } ) + preflights[candidate["id"]] = preflight families.add(decision.family) if len(selected) >= max_candidates: break @@ -622,6 +633,9 @@ def select(directory: Path, max_candidates: int, execution_file: Path | None = N (target / "candidate.json").write_text( json.dumps(candidate, indent=2, allow_nan=False) + "\n" ) + (target / "baseline-preflight.json").write_text( + preflights[candidate["id"]].model_dump_json(by_alias=True) + "\n" + ) ownership = Ownership( run_id=int(os.environ["GITHUB_RUN_ID"]), candidates=[ @@ -848,9 +862,15 @@ def main() -> int: from .lifecycle import current_session outcome = CandidateOutcome.model_validate_json(args.outcome_file.read_text()) + if ( + outcome.outcome == "failed" + and outcome.phase == "baseline" + and not outcome.reason_code + ): + raise VerificationError("Baseline failure requires a fixed reason-code") outcome = current_session().finish(outcome) (Path(os.environ["KLAUD_EVIDENCE"]) / "outcome.json").write_text( - outcome.model_dump_json(by_alias=True) + "\n" + outcome.model_dump_json(by_alias=True, exclude_unset=True) + "\n" ) return 0 if args.command == "outcome-schema": diff --git a/infx/klaud/lifecycle.py b/infx/klaud/lifecycle.py index 41a69e2a9..773a4bea8 100644 --- a/infx/klaud/lifecycle.py +++ b/infx/klaud/lifecycle.py @@ -262,7 +262,7 @@ def verify( if ( not receipt or receipt.get("head") != pull["head"]["sha"] - or receipt["outcome"] != outcome.model_dump(by_alias=True) + or receipt["outcome"] != outcome.model_dump(by_alias=True, exclude_unset=True) ): raise VerificationError("Missing verified completion report") @@ -314,7 +314,7 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: pending = self.marker.replace("klaud-outcome:", "klaud-cleanup:") request = { "head": pull["head"]["sha"], - "outcome": outcome.model_dump(by_alias=True), + "outcome": outcome.model_dump(by_alias=True, exclude_unset=True), } body = ( pending @@ -387,7 +387,7 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: } record = { "head": pull["head"]["sha"], - "outcome": outcome.model_dump(by_alias=True), + "outcome": outcome.model_dump(by_alias=True, exclude_unset=True), "validation": proof, } if self.report(pull) != record: @@ -399,18 +399,20 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: or "—" ) repairs = outcome.repairs_used if outcome.repairs_used is not None else "unknown" + reason_en = f" · Reason: {outcome.reason_code}" if outcome.reason_code else "" + reason_zh = f" · 原因：{outcome.reason_code}" if outcome.reason_code else "" body = ( self.marker + json.dumps(record) + "\n-->\n" + translated( - f"**{outcome.outcome}** · Repairs: {repairs} · Runs: {links} \nAll owned runs ended. " + f"**{outcome.outcome}**{reason_en} · Repairs: {repairs} · Runs: {links} \nAll owned runs ended. " + ( "Full sweep verified; ready for review." if proof else "PR closed; branch deleted for retry." ), - f"**{outcome.outcome}** · 修复次数：{repairs} · 运行：{links} \n所有自有运行均已结束。" + f"**{outcome.outcome}**{reason_zh} · 修复次数：{repairs} · 运行：{links} \n所有自有运行均已结束。" + ( "完整 sweep 已验证；已就绪，等待审查。" if proof diff --git a/infx/klaud/models.py b/infx/klaud/models.py index 42b9bfa5e..b3e63f831 100644 --- a/infx/klaud/models.py +++ b/infx/klaud/models.py @@ -115,6 +115,17 @@ class CandidateOutcome(Contract): pull_request: Annotated[int, Field(gt=0)] | None run_ids: list[Annotated[int, Field(gt=0)]] = Field(max_length=256) repairs_used: int | None = Field(ge=0) + reason_code: ( + Literal[ + "baseline-preflight-mismatch", + "baseline-provenance-unverified", + "baseline-point-mismatch", + "baseline-eval-unverified", + "baseline-api-unavailable", + "baseline-other", + ] + | None + ) = None @model_validator(mode="after") def consistent_outcome(self) -> CandidateOutcome: @@ -126,6 +137,8 @@ def consistent_outcome(self) -> CandidateOutcome: raise ValueError("Validated requires final-sweep evidence") if len(set(self.run_ids)) != len(self.run_ids): raise ValueError("Run IDs must be distinct") + if self.reason_code is not None and (self.outcome != "failed" or self.phase != "baseline"): + raise ValueError("Baseline reason codes require a failed baseline outcome") return self diff --git a/infx/klaud/reporting.py b/infx/klaud/reporting.py index 79deee129..40ecba743 100644 --- a/infx/klaud/reporting.py +++ b/infx/klaud/reporting.py @@ -9,16 +9,18 @@ import base64 import json +import os import re import zlib from datetime import UTC, datetime +from pathlib import Path from typing import TYPE_CHECKING, Annotated, Literal, Self from pydantic import AfterValidator, Field, field_validator, model_validator from . import github from .github import VerificationError -from .models import Contract, identity +from .models import Contract, identity, normalized_image if TYPE_CHECKING: from infx.klaud.lifecycle import Session @@ -114,6 +116,14 @@ def distinct(self) -> Self: return self +class BaselinePreflight(Contract): + candidate_id: str = Field(pattern=r"^[0-9a-f]{16}-[0-9a-f]{16}$") + base: SHA + baseline_model: str = Field(min_length=1) + source_identity: str = Field(pattern=r"^[0-9a-f]{64}$") + baseline: Baseline + + class Attempt(Contract): kind: Literal["initial", "repair", "infrastructure-retry", "final"] number: int = Field(ge=0) @@ -988,6 +998,27 @@ def resolve_baseline( def prepare_baseline(session: Session, context: dict, model: str, goal: Prose) -> Baseline: """Resolve a baseline for the current owned session.""" + evidence = Path(os.environ["KLAUD_EVIDENCE"]) + preflight_file = evidence / "baseline-preflight.json" + if preflight_file.exists(): + try: + preflight = BaselinePreflight.model_validate_json(preflight_file.read_text()) + except (OSError, ValueError): + raise VerificationError("Baseline preflight is unavailable or invalid") from None + if ( + preflight.candidate_id != session.candidate.id + or preflight.base != session.candidate.base + or preflight.baseline_model != model + or preflight.source_identity != identity(context["source"]) + or preflight.baseline.family != session.candidate.family + or preflight.baseline.date != context["source"]["date"] + or normalized_image(preflight.baseline.image) + != normalized_image(context["source"]["image"]) + ): + raise VerificationError("Baseline preflight does not match this candidate") + return preflight.baseline.model_copy(update={"goal": goal}) + if context.get("baseline-preflight-required") is True: + raise VerificationError("Required baseline preflight artifact is missing") return resolve_baseline( session.repository, session.candidate, diff --git a/utils/test_klaud_github.py b/utils/test_klaud_github.py index 0e4df36f1..cf817d1a0 100644 --- a/utils/test_klaud_github.py +++ b/utils/test_klaud_github.py @@ -3,12 +3,13 @@ import shlex import subprocess import sys +from types import SimpleNamespace import pytest from infx.klaud import __main__ as klaud from infx.klaud import api, claims, github, lifecycle, reporting, validation -from infx.klaud.models import CandidateOutcome, Feed, OwnedCandidate +from infx.klaud.models import CandidateOutcome, Feed, OwnedCandidate, identity @pytest.mark.parametrize("current_head,expected", [("ours", True), ("other", False)]) @@ -106,20 +107,24 @@ def verify(self, outcome): pull_request=None, run_ids=[], repairs_used=0, + reason_code="baseline-provenance-unverified", ) evidence = tmp_path / "evidence" evidence.mkdir() (evidence / "outcome.json").write_text( '{"outcome":"failed","phase":"baseline","pull-request":null,' - '"run-ids":[],"repairs-used":0}\n' + '"run-ids":[],"repairs-used":0,"reason-code":"baseline-provenance-unverified"}\n' ) execution = tmp_path / "execution.json" execution.write_text("{}\n") structured = tmp_path / "structured.json" structured.write_text("not json\n") output = tmp_path / "diagnostics.json" + summary = tmp_path / "summary.md" monkeypatch.setenv("KLAUD_EVIDENCE", str(evidence)) + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + monkeypatch.setenv("GITHUB_REPOSITORY", "example/project") monkeypatch.setattr(lifecycle, "current_session", Session) assert klaud.save_diagnostics(execution, structured, "success", output) @@ -132,7 +137,60 @@ def verify(self, outcome): "pull-request": None, "run-ids": [], "repairs-used": 0, + "reason-code": "baseline-provenance-unverified", } + assert "baseline-provenance-unverified" in summary.read_text() + + +def test_candidate_outcome_rejects_raw_reason_and_preserves_legacy_receipts(): + legacy = CandidateOutcome.model_validate( + { + "outcome": "failed", + "phase": "baseline", + "pull-request": None, + "run-ids": [], + "repairs-used": 0, + } + ) + assert "reason-code" not in legacy.model_dump(by_alias=True, exclude_unset=True) + with pytest.raises(ValueError): + CandidateOutcome.model_validate( + { + **legacy.model_dump(by_alias=True, exclude_unset=True), + "reason-code": "private error", + } + ) + + +def test_finish_requires_specific_baseline_reason_before_touching_session( + tmp_path, monkeypatch, capfd +): + requested = tmp_path / "requested-outcome.json" + requested.write_text( + json.dumps( + { + "outcome": "failed", + "phase": "baseline", + "pull-request": None, + "run-ids": [], + "repairs-used": 0, + } + ) + ) + monkeypatch.setattr( + sys, "argv", ["klaud", "finish", "--outcome-file", str(requested)] + ) + monkeypatch.setattr( + lifecycle, + "current_session", + lambda: pytest.fail( + "An unclassified baseline failure must not reach lifecycle cleanup" + ), + ) + assert klaud.main() == 1 + assert capfd.readouterr().out == ( + "::error::Klaud: Baseline failure requires a fixed reason-code.\n" + ) def test_recent_candidates_use_all_current_base_workflow_artifacts(monkeypatch): @@ -197,7 +255,14 @@ def test_select_continues_after_one_baseline_state_failure(tmp_path, monkeypatch def resolve(_repository, candidate, _context, _model, _goal): if candidate.id == first_id: raise OSError("transient") - return object() + return reporting.Baseline( + family=candidate.family, + date="2026-09-19", + image="example/image:1", + goal=reporting.Prose(en="Check the baseline.", zh="检查基线。"), + sources=["https://inferencex.semianalysis.com/api/v1/benchmarks"], + points=[], + ) monkeypatch.setenv("GITHUB_REPOSITORY", "example/project") monkeypatch.setenv("GITHUB_RUN_ID", "42") @@ -212,6 +277,72 @@ def resolve(_repository, candidate, _context, _model, _goal): assert selection["candidates"] == [second_id] assert selection["baseline-deferred-candidates"] == [first_id] assert selection["deferred-reason"] is None + preflight = reporting.BaselinePreflight.model_validate_json( + (tmp_path / second_id / "baseline-preflight.json").read_text() + ) + assert preflight.candidate_id == second_id + assert preflight.baseline_model == "Model" + assert preflight.source_identity == identity({}) + selected = json.loads((tmp_path / second_id / "candidate.json").read_text()) + assert selected["baseline-preflight-required"] is True + + +def test_prepare_baseline_uses_bound_preflight_and_rejects_source_drift( + tmp_path, monkeypatch +): + candidate = OwnedCandidate( + id="1" * 16 + "-" + "2" * 16, + family="configs/nvidia-master.yaml:test-family", + base="a" * 40, + ) + context = {"source": {"date": "2026-09-19", "image": "example/image:1"}} + original_goal = reporting.Prose(en="Check the baseline.", zh="检查基线。") + goal = reporting.Prose(en="Update the image.", zh="更新镜像。") + baseline = reporting.Baseline( + family=candidate.family, + date="2026-09-19", + image="example/image:1", + goal=original_goal, + sources=["https://inferencex.semianalysis.com/api/v1/benchmarks"], + points=[ + reporting.Point( + key="b" * 64, + label="8k/1k c4", + conc=4, + scenario="fixed-seq-len", + values=reporting.Values(total_tps_gpu=42), + result="passed", + ) + ], + ) + preflight = reporting.BaselinePreflight( + candidate_id=candidate.id, + base=candidate.base, + baseline_model="Model", + source_identity=identity(context["source"]), + baseline=baseline, + ) + (tmp_path / "baseline-preflight.json").write_text( + preflight.model_dump_json(by_alias=True) + ) + monkeypatch.setenv("KLAUD_EVIDENCE", str(tmp_path)) + session = SimpleNamespace(repository="example/project", candidate=candidate) + + prepared = reporting.prepare_baseline(session, context, "Model", goal) + assert prepared.goal == goal + assert prepared.points[0].values.total_tps_gpu == 42 + with pytest.raises(github.VerificationError, match="preflight does not match"): + reporting.prepare_baseline( + session, + {"source": {**context["source"], "image": "example/image:2"}}, + "Model", + goal, + ) + (tmp_path / "baseline-preflight.json").unlink() + with pytest.raises(github.VerificationError, match="artifact is missing"): + reporting.prepare_baseline( + session, {**context, "baseline-preflight-required": True}, "Model", goal + ) def test_baseline_normalizes_enroot_image_and_rejects_unverified_provenance( From 169cdffe06e56e954e553b4bd37f6f991a74d606 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:26:08 -0500 Subject: [PATCH 2/2] test: recognize Git options in SRT clone fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 让 SRT 测试夹具识别 git -c 参数后的 clone 子命令。 --- utils/test_srt_single_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/test_srt_single_node.py b/utils/test_srt_single_node.py index 979f286da..b1c31b9a2 100644 --- a/utils/test_srt_single_node.py +++ b/utils/test_srt_single_node.py @@ -278,7 +278,7 @@ def test_pool_launcher_stages_artifacts_and_propagates_failure(point, tmp_path, # Only external executables are stubbed; run the real pool launcher, shared # setup/profile/acceptance helpers, binder, and artifact collection. scripts = { - "git": 'if [[ "$1" == clone ]]; then mkdir -p "${@: -1}/configs"; else echo test-commit; fi', + "git": 'if [[ " $* " == *" clone "* ]]; then mkdir -p "${@: -1}/configs"; else echo test-commit; fi', "uv": 'if [[ "$1" == venv ]]; then mkdir -p .venv/bin; echo ":" > .venv/bin/activate; fi', "make": '[[ "$TEST_FAILURE" == bootstrap ]] && exit 13; mkdir -p bin; touch bin/uv', "squeue": '[[ "$TEST_FAILURE" == submission || "$TEST_FAILURE" == agentic ]] && echo "42"; exit 0',