Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions cmd/proven/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,9 @@ type evidence struct {
Commit string `json:"commit"`
Runs []evidenceRun `json:"runs"`
Tests []evidenceOf `json:"tests"`
// Measures is every number the jobs' tests logged for the record (proven.Measures): what
// a lane timed, on the machine the job names.
Measures []evidenceMeasure `json:"measures"`
}

// evidenceRun is one run that answers for the commit: the gate's, found by the commit's tree,
Expand All @@ -148,6 +151,17 @@ type evidenceRun struct {
// evidenceOf is one test's answer in one job.
type evidenceOf struct {
proven.Test
evidenceJob
}

// evidenceMeasure is one number a job's test logged, with the job that ran it.
type evidenceMeasure struct {
proven.Measure
evidenceJob
}

// evidenceJob is where an answer or a measurement was logged.
type evidenceJob struct {
Workflow string `json:"workflow"`
Run int64 `json:"run"`
Job string `json:"job"`
Expand Down Expand Up @@ -185,7 +199,7 @@ func tests(ctx context.Context, gh proven.GitHub, trees proven.Trees, sha, workf
}
}
}
e := evidence{Commit: sha, Runs: answering, Tests: []evidenceOf{}}
e := evidence{Commit: sha, Runs: answering, Tests: []evidenceOf{}, Measures: []evidenceMeasure{}}
for _, r := range answering {
jobs, err := gh.Jobs(ctx, r.ID)
if err != nil {
Expand All @@ -199,8 +213,12 @@ func tests(ctx context.Context, gh proven.GitHub, trees proven.Trees, sha, workf
if err != nil {
return fmt.Errorf("job %d of run %d: %w", j.ID, r.ID, err)
}
where := evidenceJob{Workflow: r.Workflow, Run: r.ID, Job: j.Name, JobURL: j.URL, Runner: j.Labels}
for _, test := range proven.Tests(log) {
e.Tests = append(e.Tests, evidenceOf{Test: test, Workflow: r.Workflow, Run: r.ID, Job: j.Name, JobURL: j.URL, Runner: j.Labels})
e.Tests = append(e.Tests, evidenceOf{Test: test, evidenceJob: where})
}
for _, m := range proven.Measures(log) {
e.Measures = append(e.Measures, evidenceMeasure{Measure: m, evidenceJob: where})
}
}
}
Expand Down
13 changes: 8 additions & 5 deletions cmd/proven/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func setup(t *testing.T) *fixture {
case "/repos/o/r/actions/runs/3/jobs":
_, _ = w.Write([]byte(`{"jobs": [{"id": 31, "name": "e2e-verify:1/1"}]}`))
case "/repos/o/r/actions/jobs/42/logs":
_, _ = w.Write([]byte("--- PASS: TestA (1.50s)\n"))
_, _ = w.Write([]byte(" measure_test.go:40: measure boot_to_running_ms=12345\n--- PASS: TestA (1.50s)\n"))
case "/repos/o/r/actions/jobs/31/logs":
_, _ = w.Write([]byte("--- PASS: TestA (1.40s)\n--- FAIL: TestB (2s)\n"))
default:
Expand Down Expand Up @@ -301,17 +301,20 @@ func TestTestsAreWhatTheAnsweringRunsLogged(t *testing.T) {
require.NoError(t, err)
var got evidence
require.NoError(t, json.Unmarshal(raw, &got))
fleet := evidenceJob{Workflow: "ci.yml", Run: 4, Job: "e2e-verify:1/1", JobURL: "j42", Runner: []string{"runs-on/fleet=kvm"}}
assert.Equal(t, evidence{
Commit: f.main[1],
Runs: []evidenceRun{
{Workflow: "ci.yml", ID: 4, URL: "u4", Conclusion: "success", Head: f.pr},
{Workflow: "kvm.yml", ID: 30, URL: "u30", Conclusion: "failure", Head: f.main[1]},
},
Tests: []evidenceOf{
{Test: proven.Test{Name: "TestA", Passed: true, Seconds: 1.5}, Workflow: "ci.yml", Run: 4, Job: "e2e-verify:1/1", JobURL: "j42", Runner: []string{"runs-on/fleet=kvm"}},
{Test: proven.Test{Name: "TestA", Passed: true, Seconds: 1.4}, Workflow: "kvm.yml", Run: 30, Job: "e2e-verify:1/1"},
{Test: proven.Test{Name: "TestB", Passed: false, Seconds: 2}, Workflow: "kvm.yml", Run: 30, Job: "e2e-verify:1/1"},
{Test: proven.Test{Name: "TestA", Passed: true, Seconds: 1.5}, evidenceJob: fleet},
{Test: proven.Test{Name: "TestA", Passed: true, Seconds: 1.4}, evidenceJob: evidenceJob{Workflow: "kvm.yml", Run: 30, Job: "e2e-verify:1/1"}},
{Test: proven.Test{Name: "TestB", Passed: false, Seconds: 2}, evidenceJob: evidenceJob{Workflow: "kvm.yml", Run: 30, Job: "e2e-verify:1/1"}},
},
// What job 42's test timed, beside the machine it ran on.
Measures: []evidenceMeasure{{Measure: proven.Measure{Name: "boot_to_running_ms", Value: 12345}, evidenceJob: fleet}},
}, got)
assert.Contains(t, f.asked, "/repos/o/r/actions/jobs/41/logs", "a job whose log is gone is asked, and says nothing")

Expand All @@ -320,7 +323,7 @@ func TestTestsAreWhatTheAnsweringRunsLogged(t *testing.T) {
f.kvm = nil
said = stdout(t, func() { err = run(t.Context(), []string{"tests", "-sha", f.main[0]}) })
require.NoError(t, err)
want, err := json.MarshalIndent(evidence{Commit: f.main[0], Runs: []evidenceRun{{Workflow: "ci.yml", ID: 6, URL: "u6", Conclusion: "failure", Head: f.main[0]}}, Tests: []evidenceOf{}}, "", " ")
want, err := json.MarshalIndent(evidence{Commit: f.main[0], Runs: []evidenceRun{{Workflow: "ci.yml", ID: 6, URL: "u6", Conclusion: "failure", Head: f.main[0]}}, Tests: []evidenceOf{}, Measures: []evidenceMeasure{}}, "", " ")
require.NoError(t, err)
assert.Equal(t, "1 runs answer for "+f.main[0]+": 0 tests logged\n"+string(want)+"\n", said)
}
Expand Down
22 changes: 22 additions & 0 deletions internal/proven/tests.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,28 @@ var pwTest = regexp.MustCompile(`([✓✘])\s+\d+\s+(?:\[[^\]]+\] › )?([^\s›
// lastTitle is the test's own title: what follows the last › of the titles.
var lastTitle = regexp.MustCompile(`^(?:.* › )?(.+)$`)

// Measure is one number a test logged for the record: "measure <name>=<value>", the unit in the
// name (boot_to_running_ms), as a lane's measuring test prints what it timed on the machine it
// ran on.
type Measure struct {
Name string `json:"name"`
Value float64 `json:"value"`
}

// measured is the line a test logs for a measurement; what the log puts before it - go test's
// file and line, a timestamp - is let through.
var measured = regexp.MustCompile(`\bmeasure ([a-z][a-z0-9_]*)=(-?[0-9]+(?:\.[0-9]+)?)\b`)

// Measures is every measurement a log holds, in the order they were logged.
func Measures(log string) []Measure {
var out []Measure
for _, m := range measured.FindAllStringSubmatch(log, -1) { // mutate-exempt: any n < 0 is every match
v, _ := strconv.ParseFloat(m[2], 64) // mutate-exempt: any bit size but 32 is 64; the expression matched a number
out = append(out, Measure{Name: m[1], Value: v})
}
return out
}

// Tests is every test a log says ended, in the order they did, each with whether it passed and
// how long it took. A test run again on a retry is there once per answer.
func Tests(log string) []Test {
Expand Down
15 changes: 15 additions & 0 deletions internal/proven/tests_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,18 @@ func TestEveryTestALogSaysEnded(t *testing.T) {
}, Tests(log))
assert.Empty(t, Tests("nothing ended here\n"))
}

// A measurement is a "measure <name>=<value>" line, however the log prefixes it; a name with a
// capital or a value with letters is not one.
func TestEveryMeasurementALogHolds(t *testing.T) {
log := "2026-10-26T13:00:01.0000000Z measure_test.go:40: measure boot_to_running_ms=12345\n" +
"=== RUN TestX\n" +
" measure_test.go:52: measure diff_etc_ms=0.5 measure snapshot_disk_ms=-1\n" +
"not a measure Boot_ms=3 nor measure disk=fast\n"
assert.Equal(t, []Measure{
{Name: "boot_to_running_ms", Value: 12345},
{Name: "diff_etc_ms", Value: 0.5},
{Name: "snapshot_disk_ms", Value: -1},
}, Measures(log))
assert.Empty(t, Measures("--- PASS: TestA (1.50s)\n"))
}
5 changes: 4 additions & 1 deletion site/src/content/docs/proven.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,10 @@ the commit's tree, as `tree` finds it, and the newest finished run of `-also` on
`release` reads it. `runs` is those runs - workflow, id, URL, conclusion and the commit each was
of - and `tests` is each test's answer in each job: `name` (a Go test's, or a Playwright test's
own title under its describe, with its `spec`), `passed`, `seconds`, and the `workflow`, `run`,
`job` and `jobUrl` that ran it, with the labels the job asked of its `runner`. A job whose log
`job` and `jobUrl` that ran it, with the labels the job asked of its `runner`. `measures` is
every number a test logged for the record - a line `measure <name>=<value>`, the unit in the
name, as a lane's measuring test prints what it timed - each with the same job and runner, so a
figure is read beside the machine it was measured on. A job whose log
the API no longer holds says nothing. The record is what a release publishes beside its files,
for a reader with no access to the repository: which test ran, where, and what it answered -
`-out` names the file, and without it the document is stdout.
Expand Down
Loading