diff --git a/cmd/proven/main.go b/cmd/proven/main.go index 08c207e..b205380 100644 --- a/cmd/proven/main.go +++ b/cmd/proven/main.go @@ -6,6 +6,7 @@ // go tool proven release exits 1 unless the gate passed on the tree and the push lanes on the commit // go tool proven times times=: each test's seconds in the last gate that passed // go tool proven flakes which failed tests are flaky (passed on the same tree) and which were red +// go tool proven tests every test the runs that answer for the commit logged, as JSON // // The repository is -repo or $GITHUB_REPOSITORY, the commit -sha or $GITHUB_SHA or HEAD, the token // $GH_TOKEN or $GITHUB_TOKEN, the API $GITHUB_API_URL or github.com's. The gate is -workflow, @@ -14,6 +15,7 @@ package main import ( "context" + "encoding/json" "errors" "flag" "fmt" @@ -33,7 +35,7 @@ func main() { } } -const usage = `usage: proven tree|base|release|times|flakes [flags] +const usage = `usage: proven tree|base|release|times|flakes|tests [flags] tree proven=true|false, whether the gate passed on the commit's tree base what a run on main is of: proven=true, or base= for mutate's diff @@ -42,6 +44,8 @@ const usage = `usage: proven tree|base|release|times|flakes [flags] flakes every test that failed in the last -limit runs of -workflow and -also: flaky when it did not fail on another run of the same tree, intermittent when on and off along a branch, red when it failed on runs in a row, once when nothing else tells + tests every test the runs that answer for the commit logged - the gate's by tree, -also's + by sha - each with its answer, its time and the job that ran it, as JSON to -out Each takes -repo, -sha, -workflow, -events; -h says the rest. ` @@ -74,6 +78,7 @@ func run(ctx context.Context, argv []string) error { depth := fs.Int("depth", 200, "base: how far back in main's first-parent history a passed run is looked for") // mutate-exempt: a default, as deep as a day on main goes also := fs.String("also", "kvm.yml", "release: the workflow that runs on every push to main, asked of the commit itself; empty is none") jobs := fs.String("jobs", "e2e-verify", "times: the jobs whose logs hold the tests' times, by the start of their name") + out := fs.String("out", "", "tests: the file the JSON is written to; stdout when empty") if err := fs.Parse(argv[1:]); err != nil { return err } @@ -114,8 +119,102 @@ func run(ctx context.Context, argv []string) error { workflows = append(workflows, *also) } return flakes(ctx, gh, trees, workflows, *limit) + case "tests": + return tests(ctx, gh, trees, *sha, *workflow, ev, *limit, *also, *out) } - return fmt.Errorf("no command %q; one of tree, base, release, times, flakes", argv[0]) + return fmt.Errorf("no command %q; one of tree, base, release, times, flakes, tests", argv[0]) +} + +// evidence is what the runs that answer for a commit logged of their tests: the record a +// release publishes beside its files, so that a reader with no access to the repository can see +// which test ran on which machine, when, and what it answered. +type evidence struct { + Commit string `json:"commit"` + Runs []evidenceRun `json:"runs"` + Tests []evidenceOf `json:"tests"` +} + +// evidenceRun is one run that answers for the commit: the gate's, found by the commit's tree, +// or the push workflow's, found by the commit itself. +type evidenceRun struct { + Workflow string `json:"workflow"` + ID int64 `json:"id"` + URL string `json:"url"` + Conclusion string `json:"conclusion"` + // Head is the commit the run was of: the gate's may be another commit with the same tree. + Head string `json:"head"` +} + +// evidenceOf is one test's answer in one job. +type evidenceOf struct { + proven.Test + Workflow string `json:"workflow"` + Run int64 `json:"run"` + Job string `json:"job"` + JobURL string `json:"jobUrl"` + // Runner is the labels the job asked of its runner; a fleet's names the machine it ran on. + Runner []string `json:"runner,omitempty"` +} + +// tests is every test the answering runs logged, each with the job it ran in: the gate's run +// on the commit's tree, as tree finds it, and the newest run of -also on the commit, as release +// does. A job whose log the API no longer holds says nothing (proven.ErrGone); the runs are in +// the record whether they logged any test or not, so a reader sees what answered. +func tests(ctx context.Context, gh proven.GitHub, trees proven.Trees, sha, workflow string, events []string, limit int, also, out string) error { + t, err := trees(ctx, sha) + if err != nil { + return err + } + runs, err := gh.Runs(ctx, workflow, completed, limit) + if err != nil { + return err + } + var answering []evidenceRun + if answer, found := proven.Answer(ctx, runs, sha, t, events, trees); found { + answering = append(answering, evidenceRun{Workflow: workflow, ID: answer.ID, URL: answer.URL, Conclusion: answer.Conclusion, Head: answer.HeadSHA}) + } + if also != "" { + alsoRuns, err := gh.Runs(ctx, also, url.Values{"head_sha": {sha}}, limit) + if err != nil { + return err + } + for _, r := range alsoRuns { + if r.Status == "completed" && r.Conclusion != "cancelled" { + answering = append(answering, evidenceRun{Workflow: also, ID: r.ID, URL: r.URL, Conclusion: r.Conclusion, Head: r.HeadSHA}) + break + } + } + } + e := evidence{Commit: sha, Runs: answering, Tests: []evidenceOf{}} + for _, r := range answering { + jobs, err := gh.Jobs(ctx, r.ID) + if err != nil { + return fmt.Errorf("run %d: %w", r.ID, err) + } + for _, j := range jobs { + log, err := gh.Log(ctx, j.ID) + if errors.Is(err, proven.ErrGone) { + continue + } + if err != nil { + return fmt.Errorf("job %d of run %d: %w", j.ID, r.ID, err) + } + for _, test := range proven.Tests(log) { + e.Tests = append(e.Tests, evidenceOf{Test: test, Workflow: r.Workflow, Run: r.ID, Job: j.Name, JobURL: j.URL, Runner: j.Labels}) + } + } + } + b, err := json.MarshalIndent(e, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + fmt.Printf("%d runs answer for %s: %d tests logged\n", len(e.Runs), sha, len(e.Tests)) + if out == "" { + _, err = os.Stdout.Write(b) + return err + } + return os.WriteFile(out, b, 0o600) } // tree answers whether the gate passed on the commit's tree, for a workflow that then runs only diff --git a/cmd/proven/main_test.go b/cmd/proven/main_test.go index 1af1c71..91ffc2e 100644 --- a/cmd/proven/main_test.go +++ b/cmd/proven/main_test.go @@ -15,6 +15,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/spin-stack/go-tools/internal/proven" ) // stdout is what f printed to os.Stdout. @@ -97,7 +99,9 @@ func setup(t *testing.T) *fixture { case "/repos/o/r/actions/workflows/kvm.yml/runs": _ = json.NewEncoder(w).Encode(map[string]any{"workflow_runs": f.kvm}) case "/repos/o/r/actions/runs/4/jobs": - _, _ = w.Write([]byte(`{"jobs": [{"id": 41, "name": "short"}, {"id": 42, "name": "e2e-verify:1/1"}]}`)) + _, _ = w.Write([]byte(`{"jobs": [{"id": 41, "name": "short"}, {"id": 42, "name": "e2e-verify:1/1", "html_url": "j42", "labels": ["runs-on/fleet=kvm"]}]}`)) + case "/repos/o/r/actions/runs/6/jobs": + _, _ = w.Write([]byte(`{"jobs": []}`)) case "/repos/o/r/actions/runs/3/jobs": _, _ = w.Write([]byte(`{"jobs": [{"id": 31, "name": "e2e-verify:1/1"}]}`)) case "/repos/o/r/actions/jobs/42/logs": @@ -278,6 +282,49 @@ func TestTimesAreTheLastGatesThatLoggedAny(t *testing.T) { assert.Equal(t, "run none: 0 tests' times\ntimes=\n", out) } +// tests: main~1 is answered for by the pull request's run 4 (its tree) and kvm's run 30 (its +// sha); every test their jobs logged is in the record with its job, a job whose log is gone says +// nothing, and the record goes to -out or stdout. +func TestTestsAreWhatTheAnsweringRunsLogged(t *testing.T) { + f := setup(t) + // A re-run still going is not an answer yet; the newest that finished is. + f.kvm = []ciRun{ + {ID: 33, HeadSHA: f.main[1], HeadBranch: "main", Event: "push", Status: "in_progress", URL: "u33"}, + {ID: 30, HeadSHA: f.main[1], HeadBranch: "main", Event: "push", Status: "completed", Conclusion: "failure", URL: "u30"}, + } + out := filepath.Join(t.TempDir(), "tests.json") + var err error + said := stdout(t, func() { err = run(t.Context(), []string{"tests", "-sha", f.main[1], "-out", out}) }) + require.NoError(t, err) + assert.Equal(t, "2 runs answer for "+f.main[1]+": 3 tests logged\n", said) + raw, err := os.ReadFile(out) + require.NoError(t, err) + var got evidence + require.NoError(t, json.Unmarshal(raw, &got)) + assert.Equal(t, evidence{ + Commit: f.main[1], + Runs: []evidenceRun{ + {Workflow: "ci.yml", ID: 4, URL: "u4", Conclusion: "success", Head: f.pr}, + {Workflow: "kvm.yml", ID: 30, URL: "u30", Conclusion: "failure", Head: f.main[1]}, + }, + Tests: []evidenceOf{ + {Test: proven.Test{Name: "TestA", Passed: true, Seconds: 1.5}, Workflow: "ci.yml", Run: 4, Job: "e2e-verify:1/1", JobURL: "j42", Runner: []string{"runs-on/fleet=kvm"}}, + {Test: proven.Test{Name: "TestA", Passed: true, Seconds: 1.4}, Workflow: "kvm.yml", Run: 30, Job: "e2e-verify:1/1"}, + {Test: proven.Test{Name: "TestB", Passed: false, Seconds: 2}, Workflow: "kvm.yml", Run: 30, Job: "e2e-verify:1/1"}, + }, + }, got) + assert.Contains(t, f.asked, "/repos/o/r/actions/jobs/41/logs", "a job whose log is gone is asked, and says nothing") + + // main's head: the gate's run 6 failed on its tree and logged nothing, and kvm never ran on + // it - a record of that run and no test, to stdout. + f.kvm = nil + said = stdout(t, func() { err = run(t.Context(), []string{"tests", "-sha", f.main[0]}) }) + require.NoError(t, err) + want, err := json.MarshalIndent(evidence{Commit: f.main[0], Runs: []evidenceRun{{Workflow: "ci.yml", ID: 6, URL: "u6", Conclusion: "failure", Head: f.main[0]}}, Tests: []evidenceOf{}}, "", " ") + require.NoError(t, err) + assert.Equal(t, "1 runs answer for "+f.main[0]+": 0 tests logged\n"+string(want)+"\n", said) +} + // flakes: run 6 failed TestA on main's tree, which no other run has - red; kvm's TestB failed on // two runs of main~1's tree and passed on a third - flaky, and a quarantine candidate. The table // goes to the summary when there is one. diff --git a/internal/proven/github.go b/internal/proven/github.go index 6d584a9..cc56c61 100644 --- a/internal/proven/github.go +++ b/internal/proven/github.go @@ -59,6 +59,9 @@ type Job struct { ID int64 `json:"id"` Name string `json:"name"` Conclusion string `json:"conclusion"` // success, failure, cancelled, skipped; empty until completed + URL string `json:"html_url"` + // Labels is what the job asked of its runner: a fleet's label names the kind of machine it ran on. + Labels []string `json:"labels"` } // Artifact is one file a run kept: a lane's logs, when it failed. diff --git a/internal/proven/tests.go b/internal/proven/tests.go new file mode 100644 index 0000000..968757d --- /dev/null +++ b/internal/proven/tests.go @@ -0,0 +1,56 @@ +package proven + +import ( + "regexp" + "strconv" +) + +// Test is one test a job's log says ended, with its answer: a Go test by go test's own line for +// it, a Playwright test by its list reporter's. +type Test struct { + // Name is the Go test's name, or the Playwright test's title: the last of its titles, since + // the ones before it are its describe blocks'. + Name string `json:"name"` + // Spec is the file a Playwright test is in, by its name alone; empty for a Go test. + Spec string `json:"spec,omitempty"` + Passed bool `json:"passed"` + Seconds float64 `json:"seconds"` +} + +// goTest is go test's line for a top-level test that ended: a subtest's name has a slash in it +// and is its parent's business. What a CI log puts before it on the line - a timestamp - is let +// through. +var goTest = regexp.MustCompile(`--- (PASS|FAIL): (Test[A-Za-z0-9_]+) \(([0-9.]+)s\)`) + +// pwTest is Playwright's list reporter's line for a test that ended, as it prints it with no +// terminal: the mark, the test's number, the project, the spec with its line and column, the +// titles joined by › (a describe's, then the test's), a retry's suffix, and how long it took. +var pwTest = regexp.MustCompile(`([✓✘])\s+\d+\s+(?:\[[^\]]+\] › )?([^\s›]+\.[a-z]+):\d+:\d+ › (.+?)(?: \(retry #\d+\))? \(([0-9.]+)(ms|s|m)\)\s*$`) + +// lastTitle is the test's own title: what follows the last › of the titles. +var lastTitle = regexp.MustCompile(`^(?:.* › )?(.+)$`) + +// Tests is every test a log says ended, in the order they did, each with whether it passed and +// how long it took. A test run again on a retry is there once per answer. +func Tests(log string) []Test { + var out []Test + for _, m := range goTest.FindAllStringSubmatch(log, -1) { // mutate-exempt: any n < 0 is every match + seconds, _ := strconv.ParseFloat(m[3], 64) // the expression matched digits and dots; a second dot is 0, which is the log's mistake to show + out = append(out, Test{Name: m[2], Passed: m[1] == "PASS", Seconds: seconds}) + } + for _, line := range regexp.MustCompile(`\r?\n`).Split(log, -1) { // mutate-exempt: any n < 0 is every line + m := pwTest.FindStringSubmatch(line) + if m == nil { + continue + } + n, _ := strconv.ParseFloat(m[4], 64) // mutate-exempt: any bit size but 32 is 64; the digits and dots matched, as above + switch m[5] { + case "ms": + n /= 1000 + case "m": + n *= 60 + } + out = append(out, Test{Name: lastTitle.FindStringSubmatch(m[3])[1], Spec: m[2], Passed: m[1] == "✓", Seconds: n}) + } + return out +} diff --git a/internal/proven/tests_test.go b/internal/proven/tests_test.go new file mode 100644 index 0000000..7910c66 --- /dev/null +++ b/internal/proven/tests_test.go @@ -0,0 +1,31 @@ +package proven + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Every test a log says ended, Go's and Playwright's alike, with its answer and its time: a +// subtest is its parent's, a Playwright test is named by its own title under its describe, a +// retry is one more answer, and what the log puts before a line does not hide it. +func TestEveryTestALogSaysEnded(t *testing.T) { + log := "2026-10-26T13:00:00.0000000Z === RUN TestA\n" + + "2026-10-26T13:00:01.0000000Z --- PASS: TestA/one (0.10s)\n" + + "2026-10-26T13:00:01.0000000Z --- PASS: TestA (1.50s)\n" + + "2026-10-26T13:00:03.0000000Z --- FAIL: TestB (2s)\n" + + "2026-10-26T13:00:04.0000000Z ✓ 1 [chromium] › auth.spec.ts:30:3 › the CLI's device page stands alone (3.2s)\n" + + "2026-10-26T13:00:05.0000000Z ✘ 2 [chromium] › admin.spec.ts:40:3 › presets › an admin's preset sizes a workspace (retry #1) (850ms)\n" + + "2026-10-26T13:00:06.0000000Z ✓ 3 [chromium] › admin.spec.ts:40:3 › presets › an admin's preset sizes a workspace (1.1m)\n" + + "2026-10-26T13:00:07.0000000Z - 4 [chromium] › skipped.spec.ts:1:1 › nothing here\n" + + "2026-10-26T13:00:08.0000000Z ✓ 5 site.spec.ts:12:3 › a person's workspaces (2s)\n" + assert.Equal(t, []Test{ + {Name: "TestA", Passed: true, Seconds: 1.5}, + {Name: "TestB", Passed: false, Seconds: 2}, + {Name: "the CLI's device page stands alone", Spec: "auth.spec.ts", Passed: true, Seconds: 3.2}, + {Name: "an admin's preset sizes a workspace", Spec: "admin.spec.ts", Passed: false, Seconds: 0.85}, + {Name: "an admin's preset sizes a workspace", Spec: "admin.spec.ts", Passed: true, Seconds: 66}, + {Name: "a person's workspaces", Spec: "site.spec.ts", Passed: true, Seconds: 2}, + }, Tests(log)) + assert.Empty(t, Tests("nothing ended here\n")) +} diff --git a/site/src/content/docs/proven.md b/site/src/content/docs/proven.md index be10f50..8596063 100644 --- a/site/src/content/docs/proven.md +++ b/site/src/content/docs/proven.md @@ -1,6 +1,6 @@ --- title: proven -description: Asks GitHub what CI already proved about a commit - which run answers for its tree, what a run on main is of, whether a release may be made, how long each test took, which failed tests are flaky - and writes the answer where the workflow reads it. +description: Asks GitHub what CI already proved about a commit - which run answers for its tree, what a run on main is of, whether a release may be made, how long each test took, which failed tests are flaky, what every test answered - and writes the answer where the workflow reads it. --- A workflow that decides something from what GitHub says - which commit to diff against, whether @@ -11,7 +11,7 @@ commits, the lane timed out, and nothing on main could pass again until somebody `proven` is those functions in Go, with their tables, and one small client for the API. ```sh -go tool proven tree|base|release|times|flakes [flags] +go tool proven tree|base|release|times|flakes|tests [flags] ``` Every command takes the repository from `-repo` or `$GITHUB_REPOSITORY`, the commit from `-sha` @@ -93,6 +93,24 @@ names start with `-jobs` (`e2e-verify`), from the newest run that passed and log proven by another ran no lane. A split of the tests over machines reads it, so the shards are even by measured time rather than by count; with none, it splits by name. +## tests + +```yaml title=".github/workflows/release.yml" +- env: + GH_TOKEN: ${{ github.token }} + run: go tool task release:evidence # go tool proven tests -out dist/evidence.json +``` + +Every test the runs that answer for the commit logged, as one JSON document: the gate's run on +the commit's tree, as `tree` finds it, and the newest finished run of `-also` on the commit, as +`release` reads it. `runs` is those runs - workflow, id, URL, conclusion and the commit each was +of - and `tests` is each test's answer in each job: `name` (a Go test's, or a Playwright test's +own title under its describe, with its `spec`), `passed`, `seconds`, and the `workflow`, `run`, +`job` and `jobUrl` that ran it, with the labels the job asked of its `runner`. A job whose log +the API no longer holds says nothing. The record is what a release publishes beside its files, +for a reader with no access to the repository: which test ran, where, and what it answered - +`-out` names the file, and without it the document is stdout. + ## flakes A test that failed and did not fail on another run of the same tree is a flake by definition: the