proven: tests, every test the answering runs logged, as a release's evidence

cmd/proven/main.go

@@ -6,6 +6,7 @@
 //	go tool proven release   exits 1 unless the gate passed on the tree and the push lanes on the commit
 //	go tool proven times     times=<lines>: each test's seconds in the last gate that passed
 //	go tool proven flakes    which failed tests are flaky (passed on the same tree) and which were red
+//	go tool proven tests     every test the runs that answer for the commit logged, as JSON
 //
 // The repository is -repo or $GITHUB_REPOSITORY, the commit -sha or $GITHUB_SHA or HEAD, the token
 // $GH_TOKEN or $GITHUB_TOKEN, the API $GITHUB_API_URL or github.com's. The gate is -workflow,
@@ -14,6 +15,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -33,7 +35,7 @@ func main() {
 	}
 }
 
-const usage = `usage: proven tree|base|release|times|flakes [flags]
+const usage = `usage: proven tree|base|release|times|flakes|tests [flags]
 
   tree     proven=true|false, whether the gate passed on the commit's tree
   base     what a run on main is of: proven=true, or base=<commit> for mutate's diff
@@ -42,6 +44,8 @@ const usage = `usage: proven tree|base|release|times|flakes [flags]
   flakes   every test that failed in the last -limit runs of -workflow and -also: flaky when it
            did not fail on another run of the same tree, intermittent when on and off along a
            branch, red when it failed on runs in a row, once when nothing else tells
+  tests    every test the runs that answer for the commit logged - the gate's by tree, -also's
+           by sha - each with its answer, its time and the job that ran it, as JSON to -out
 
 Each takes -repo, -sha, -workflow, -events; -h says the rest.
 `
@@ -74,6 +78,7 @@ func run(ctx context.Context, argv []string) error {
 	depth := fs.Int("depth", 200, "base: how far back in main's first-parent history a passed run is looked for") // mutate-exempt: a default, as deep as a day on main goes
 	also := fs.String("also", "kvm.yml", "release: the workflow that runs on every push to main, asked of the commit itself; empty is none")
 	jobs := fs.String("jobs", "e2e-verify", "times: the jobs whose logs hold the tests' times, by the start of their name")
+	out := fs.String("out", "", "tests: the file the JSON is written to; stdout when empty")
 	if err := fs.Parse(argv[1:]); err != nil {
 		return err
 	}
@@ -114,8 +119,102 @@ func run(ctx context.Context, argv []string) error {
 			workflows = append(workflows, *also)
 		}
 		return flakes(ctx, gh, trees, workflows, *limit)
+	case "tests":
+		return tests(ctx, gh, trees, *sha, *workflow, ev, *limit, *also, *out)
 	}
-	return fmt.Errorf("no command %q; one of tree, base, release, times, flakes", argv[0])
+	return fmt.Errorf("no command %q; one of tree, base, release, times, flakes, tests", argv[0])
+}
+
+// evidence is what the runs that answer for a commit logged of their tests: the record a
+// release publishes beside its files, so that a reader with no access to the repository can see
+// which test ran on which machine, when, and what it answered.
+type evidence struct {
+	Commit string        `json:"commit"`
+	Runs   []evidenceRun `json:"runs"`
+	Tests  []evidenceOf  `json:"tests"`
+}
+
+// evidenceRun is one run that answers for the commit: the gate's, found by the commit's tree,
+// or the push workflow's, found by the commit itself.
+type evidenceRun struct {
+	Workflow   string `json:"workflow"`
+	ID         int64  `json:"id"`
+	URL        string `json:"url"`
+	Conclusion string `json:"conclusion"`
+	// Head is the commit the run was of: the gate's may be another commit with the same tree.
+	Head string `json:"head"`
+}
+
+// evidenceOf is one test's answer in one job.
+type evidenceOf struct {
+	proven.Test
+	Workflow string `json:"workflow"`
+	Run      int64  `json:"run"`
+	Job      string `json:"job"`
+	JobURL   string `json:"jobUrl"`
+	// Runner is the labels the job asked of its runner; a fleet's names the machine it ran on.
+	Runner []string `json:"runner,omitempty"`
+}
+
+// tests is every test the answering runs logged, each with the job it ran in: the gate's run
+// on the commit's tree, as tree finds it, and the newest run of -also on the commit, as release
+// does. A job whose log the API no longer holds says nothing (proven.ErrGone); the runs are in
+// the record whether they logged any test or not, so a reader sees what answered.
+func tests(ctx context.Context, gh proven.GitHub, trees proven.Trees, sha, workflow string, events []string, limit int, also, out string) error {
+	t, err := trees(ctx, sha)
+	if err != nil {
+		return err
+	}
+	runs, err := gh.Runs(ctx, workflow, completed, limit)
+	if err != nil {
+		return err
+	}
+	var answering []evidenceRun
+	if answer, found := proven.Answer(ctx, runs, sha, t, events, trees); found {
+		answering = append(answering, evidenceRun{Workflow: workflow, ID: answer.ID, URL: answer.URL, Conclusion: answer.Conclusion, Head: answer.HeadSHA})
+	}
+	if also != "" {
+		alsoRuns, err := gh.Runs(ctx, also, url.Values{"head_sha": {sha}}, limit)
+		if err != nil {
+			return err
+		}
+		for _, r := range alsoRuns {
+			if r.Status == "completed" && r.Conclusion != "cancelled" {
+				answering = append(answering, evidenceRun{Workflow: also, ID: r.ID, URL: r.URL, Conclusion: r.Conclusion, Head: r.HeadSHA})
+				break
+			}
+		}
+	}
+	e := evidence{Commit: sha, Runs: answering, Tests: []evidenceOf{}}
+	for _, r := range answering {
+		jobs, err := gh.Jobs(ctx, r.ID)
+		if err != nil {
+			return fmt.Errorf("run %d: %w", r.ID, err)
+		}
+		for _, j := range jobs {
+			log, err := gh.Log(ctx, j.ID)
+			if errors.Is(err, proven.ErrGone) {
+				continue
+			}
+			if err != nil {
+				return fmt.Errorf("job %d of run %d: %w", j.ID, r.ID, err)
+			}
+			for _, test := range proven.Tests(log) {
+				e.Tests = append(e.Tests, evidenceOf{Test: test, Workflow: r.Workflow, Run: r.ID, Job: j.Name, JobURL: j.URL, Runner: j.Labels})
+			}
+		}
+	}
+	b, err := json.MarshalIndent(e, "", "  ")
+	if err != nil {
+		return err
+	}
+	b = append(b, '\n')
+	fmt.Printf("%d runs answer for %s: %d tests logged\n", len(e.Runs), sha, len(e.Tests))
+	if out == "" {
+		_, err = os.Stdout.Write(b)
+		return err
+	}
+	return os.WriteFile(out, b, 0o600)
 }
 
 // tree answers whether the gate passed on the commit's tree, for a workflow that then runs only

cmd/proven/main_test.go

@@ -15,6 +15,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/spin-stack/go-tools/internal/proven"
 )
 
 // stdout is what f printed to os.Stdout.
@@ -97,7 +99,9 @@ func setup(t *testing.T) *fixture {
 		case "/repos/o/r/actions/workflows/kvm.yml/runs":
 			_ = json.NewEncoder(w).Encode(map[string]any{"workflow_runs": f.kvm})
 		case "/repos/o/r/actions/runs/4/jobs":
-			_, _ = w.Write([]byte(`{"jobs": [{"id": 41, "name": "short"}, {"id": 42, "name": "e2e-verify:1/1"}]}`))
+			_, _ = w.Write([]byte(`{"jobs": [{"id": 41, "name": "short"}, {"id": 42, "name": "e2e-verify:1/1", "html_url": "j42", "labels": ["runs-on/fleet=kvm"]}]}`))
+		case "/repos/o/r/actions/runs/6/jobs":
+			_, _ = w.Write([]byte(`{"jobs": []}`))
 		case "/repos/o/r/actions/runs/3/jobs":
 			_, _ = w.Write([]byte(`{"jobs": [{"id": 31, "name": "e2e-verify:1/1"}]}`))
 		case "/repos/o/r/actions/jobs/42/logs":
@@ -278,6 +282,49 @@ func TestTimesAreTheLastGatesThatLoggedAny(t *testing.T) {
 	assert.Equal(t, "run none: 0 tests' times\ntimes=\n", out)
 }
 
+// tests: main~1 is answered for by the pull request's run 4 (its tree) and kvm's run 30 (its
+// sha); every test their jobs logged is in the record with its job, a job whose log is gone says
+// nothing, and the record goes to -out or stdout.
+func TestTestsAreWhatTheAnsweringRunsLogged(t *testing.T) {
+	f := setup(t)
+	// A re-run still going is not an answer yet; the newest that finished is.
+	f.kvm = []ciRun{
+		{ID: 33, HeadSHA: f.main[1], HeadBranch: "main", Event: "push", Status: "in_progress", URL: "u33"},
+		{ID: 30, HeadSHA: f.main[1], HeadBranch: "main", Event: "push", Status: "completed", Conclusion: "failure", URL: "u30"},
+	}
+	out := filepath.Join(t.TempDir(), "tests.json")
+	var err error
+	said := stdout(t, func() { err = run(t.Context(), []string{"tests", "-sha", f.main[1], "-out", out}) })
+	require.NoError(t, err)
+	assert.Equal(t, "2 runs answer for "+f.main[1]+": 3 tests logged\n", said)
+	raw, err := os.ReadFile(out)
+	require.NoError(t, err)
+	var got evidence
+	require.NoError(t, json.Unmarshal(raw, &got))
+	assert.Equal(t, evidence{
+		Commit: f.main[1],
+		Runs: []evidenceRun{
+			{Workflow: "ci.yml", ID: 4, URL: "u4", Conclusion: "success", Head: f.pr},
+			{Workflow: "kvm.yml", ID: 30, URL: "u30", Conclusion: "failure", Head: f.main[1]},
+		},
+		Tests: []evidenceOf{
+			{Test: proven.Test{Name: "TestA", Passed: true, Seconds: 1.5}, Workflow: "ci.yml", Run: 4, Job: "e2e-verify:1/1", JobURL: "j42", Runner: []string{"runs-on/fleet=kvm"}},
+			{Test: proven.Test{Name: "TestA", Passed: true, Seconds: 1.4}, Workflow: "kvm.yml", Run: 30, Job: "e2e-verify:1/1"},
+			{Test: proven.Test{Name: "TestB", Passed: false, Seconds: 2}, Workflow: "kvm.yml", Run: 30, Job: "e2e-verify:1/1"},
+		},
+	}, got)
+	assert.Contains(t, f.asked, "/repos/o/r/actions/jobs/41/logs", "a job whose log is gone is asked, and says nothing")
+
+	// main's head: the gate's run 6 failed on its tree and logged nothing, and kvm never ran on
+	// it - a record of that run and no test, to stdout.
+	f.kvm = nil
+	said = stdout(t, func() { err = run(t.Context(), []string{"tests", "-sha", f.main[0]}) })
+	require.NoError(t, err)
+	want, err := json.MarshalIndent(evidence{Commit: f.main[0], Runs: []evidenceRun{{Workflow: "ci.yml", ID: 6, URL: "u6", Conclusion: "failure", Head: f.main[0]}}, Tests: []evidenceOf{}}, "", "  ")
+	require.NoError(t, err)
+	assert.Equal(t, "1 runs answer for "+f.main[0]+": 0 tests logged\n"+string(want)+"\n", said)
+}
+
 // flakes: run 6 failed TestA on main's tree, which no other run has - red; kvm's TestB failed on
 // two runs of main~1's tree and passed on a third - flaky, and a quarantine candidate. The table
 // goes to the summary when there is one.

internal/proven/github.go

@@ -59,6 +59,9 @@ type Job struct {
 	ID         int64  `json:"id"`
 	Name       string `json:"name"`
 	Conclusion string `json:"conclusion"` // success, failure, cancelled, skipped; empty until completed
+	URL        string `json:"html_url"`
+	// Labels is what the job asked of its runner: a fleet's label names the kind of machine it ran on.
+	Labels []string `json:"labels"`
 }
 
 // Artifact is one file a run kept: a lane's logs, when it failed.

internal/proven/tests.go

@@ -0,0 +1,56 @@
+package proven
+
+import (
+	"regexp"
+	"strconv"
+)
+
+// Test is one test a job's log says ended, with its answer: a Go test by go test's own line for
+// it, a Playwright test by its list reporter's.
+type Test struct {
+	// Name is the Go test's name, or the Playwright test's title: the last of its titles, since
+	// the ones before it are its describe blocks'.
+	Name string `json:"name"`
+	// Spec is the file a Playwright test is in, by its name alone; empty for a Go test.
+	Spec    string  `json:"spec,omitempty"`
+	Passed  bool    `json:"passed"`
+	Seconds float64 `json:"seconds"`
+}
+
+// goTest is go test's line for a top-level test that ended: a subtest's name has a slash in it
+// and is its parent's business. What a CI log puts before it on the line - a timestamp - is let
+// through.
+var goTest = regexp.MustCompile(`--- (PASS|FAIL): (Test[A-Za-z0-9_]+) \(([0-9.]+)s\)`)
+
+// pwTest is Playwright's list reporter's line for a test that ended, as it prints it with no
+// terminal: the mark, the test's number, the project, the spec with its line and column, the
+// titles joined by › (a describe's, then the test's), a retry's suffix, and how long it took.
+var pwTest = regexp.MustCompile(`([✓✘])\s+\d+\s+(?:\[[^\]]+\] › )?([^\s›]+\.[a-z]+):\d+:\d+ › (.+?)(?: \(retry #\d+\))? \(([0-9.]+)(ms|s|m)\)\s*$`)
+
+// lastTitle is the test's own title: what follows the last › of the titles.
+var lastTitle = regexp.MustCompile(`^(?:.* › )?(.+)$`)
+
+// Tests is every test a log says ended, in the order they did, each with whether it passed and
+// how long it took. A test run again on a retry is there once per answer.
+func Tests(log string) []Test {
+	var out []Test
+	for _, m := range goTest.FindAllStringSubmatch(log, -1) { // mutate-exempt: any n < 0 is every match
+		seconds, _ := strconv.ParseFloat(m[3], 64) // the expression matched digits and dots; a second dot is 0, which is the log's mistake to show
+		out = append(out, Test{Name: m[2], Passed: m[1] == "PASS", Seconds: seconds})
+	}
+	for _, line := range regexp.MustCompile(`\r?\n`).Split(log, -1) { // mutate-exempt: any n < 0 is every line
+		m := pwTest.FindStringSubmatch(line)
+		if m == nil {
+			continue
+		}
+		n, _ := strconv.ParseFloat(m[4], 64) // mutate-exempt: any bit size but 32 is 64; the digits and dots matched, as above
+		switch m[5] {
+		case "ms":
+			n /= 1000
+		case "m":
+			n *= 60
+		}
+		out = append(out, Test{Name: lastTitle.FindStringSubmatch(m[3])[1], Spec: m[2], Passed: m[1] == "✓", Seconds: n})
+	}
+	return out
+}

internal/proven/tests_test.go

@@ -0,0 +1,31 @@
+package proven
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Every test a log says ended, Go's and Playwright's alike, with its answer and its time: a
+// subtest is its parent's, a Playwright test is named by its own title under its describe, a
+// retry is one more answer, and what the log puts before a line does not hide it.
+func TestEveryTestALogSaysEnded(t *testing.T) {
+	log := "2026-10-26T13:00:00.0000000Z === RUN   TestA\n" +
+		"2026-10-26T13:00:01.0000000Z     --- PASS: TestA/one (0.10s)\n" +
+		"2026-10-26T13:00:01.0000000Z --- PASS: TestA (1.50s)\n" +
+		"2026-10-26T13:00:03.0000000Z --- FAIL: TestB (2s)\n" +
+		"2026-10-26T13:00:04.0000000Z   ✓  1 [chromium] › auth.spec.ts:30:3 › the CLI's device page stands alone (3.2s)\n" +
+		"2026-10-26T13:00:05.0000000Z   ✘  2 [chromium] › admin.spec.ts:40:3 › presets › an admin's preset sizes a workspace (retry #1) (850ms)\n" +
+		"2026-10-26T13:00:06.0000000Z   ✓  3 [chromium] › admin.spec.ts:40:3 › presets › an admin's preset sizes a workspace (1.1m)\n" +
+		"2026-10-26T13:00:07.0000000Z   -  4 [chromium] › skipped.spec.ts:1:1 › nothing here\n" +
+		"2026-10-26T13:00:08.0000000Z   ✓  5 site.spec.ts:12:3 › a person's workspaces (2s)\n"
+	assert.Equal(t, []Test{
+		{Name: "TestA", Passed: true, Seconds: 1.5},
+		{Name: "TestB", Passed: false, Seconds: 2},
+		{Name: "the CLI's device page stands alone", Spec: "auth.spec.ts", Passed: true, Seconds: 3.2},
+		{Name: "an admin's preset sizes a workspace", Spec: "admin.spec.ts", Passed: false, Seconds: 0.85},
+		{Name: "an admin's preset sizes a workspace", Spec: "admin.spec.ts", Passed: true, Seconds: 66},
+		{Name: "a person's workspaces", Spec: "site.spec.ts", Passed: true, Seconds: 2},
+	}, Tests(log))
+	assert.Empty(t, Tests("nothing ended here\n"))
+}

site/src/content/docs/proven.md

@@ -1,6 +1,6 @@
 ---
 title: proven
-description: Asks GitHub what CI already proved about a commit - which run answers for its tree, what a run on main is of, whether a release may be made, how long each test took, which failed tests are flaky - and writes the answer where the workflow reads it.
+description: Asks GitHub what CI already proved about a commit - which run answers for its tree, what a run on main is of, whether a release may be made, how long each test took, which failed tests are flaky, what every test answered - and writes the answer where the workflow reads it.
 ---
 
 A workflow that decides something from what GitHub says - which commit to diff against, whether
@@ -11,7 +11,7 @@ commits, the lane timed out, and nothing on main could pass again until somebody
 `proven` is those functions in Go, with their tables, and one small client for the API.
 
 ```sh
-go tool proven tree|base|release|times|flakes [flags]
+go tool proven tree|base|release|times|flakes|tests [flags]
 ```
 
 Every command takes the repository from `-repo` or `$GITHUB_REPOSITORY`, the commit from `-sha`
@@ -93,6 +93,24 @@ names start with `-jobs` (`e2e-verify`), from the newest run that passed and log
 proven by another ran no lane. A split of the tests over machines reads it, so the shards are even
 by measured time rather than by count; with none, it splits by name.
 
+## tests
+
+```yaml title=".github/workflows/release.yml"
+- env:
+    GH_TOKEN: ${{ github.token }}
+  run: go tool task release:evidence   # go tool proven tests -out dist/evidence.json
+```
+
+Every test the runs that answer for the commit logged, as one JSON document: the gate's run on
+the commit's tree, as `tree` finds it, and the newest finished run of `-also` on the commit, as
+`release` reads it. `runs` is those runs - workflow, id, URL, conclusion and the commit each was
+of - and `tests` is each test's answer in each job: `name` (a Go test's, or a Playwright test's
+own title under its describe, with its `spec`), `passed`, `seconds`, and the `workflow`, `run`,
+`job` and `jobUrl` that ran it, with the labels the job asked of its `runner`. A job whose log
+the API no longer holds says nothing. The record is what a release publishes beside its files,
+for a reader with no access to the repository: which test ran, where, and what it answered -
+`-out` names the file, and without it the document is stdout.
+
 ## flakes
 
 A test that failed and did not fail on another run of the same tree is a flake by definition: the