From fa010154c1864d760096bcec441ad350ad79754d Mon Sep 17 00:00:00 2001 From: Carlos Fernandez Date: Sun, 16 Aug 2026 22:44:19 -0700 Subject: [PATCH 1/2] =?UTF-8?q?feat(run):=20add=20`sp=20run=20report`=20?= =?UTF-8?q?=E2=80=94=20the=20verdict,=20then=20who=20caused=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run's verdict answers one question: did the output match the approved file. Reviewers need a second one, and it is the one that decides whether a branch is blocked -- did *this change* cause it. Answering that today means finding a baseline by hand, running `run compare` against it, and knowing which baseline was the right choice. `sp run report ` does both. It prints the verdict as-is, resolves the runs worth comparing against -- the newest on the target branch, and the newest that predates this run -- and reports each failure against them. A failure present in both is not this change's doing; one the reference passed is where to start. The second reference is a proxy for the branch point, not ancestry: the API exposes no commit graph, so ordering by time is the best available and the label says "before this run" rather than "ancestor". `--against` names a baseline explicitly when the caller knows better. Table output caps each list at REPORT_LIST_LIMIT rows and says how many were held back; JSON is never truncated, so a script loses nothing. --- AGENTS.md | 2 + README.md | 1 + sp_cli/commands/run.py | 142 ++++++++++++++++++++++++++++++++++++++++- sp_cli/compare.py | 58 +++++++++++++++++ sp_cli/constants.py | 5 ++ tests/test_cli.py | 110 +++++++++++++++++++++++++++++++ 6 files changed, 316 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 72b91e5..349e4fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,6 +59,8 @@ Then drill in: ```bash sp run compare # which failures are new vs the baseline +sp run report # verdict + the same diff against resolved + # references; the "is this mine?" answer sp run summary # counts only, cheapest sp run error-summary # grouped error counts, server-derived sp run result # one test: exit code, command, outputs diff --git a/README.md b/README.md index c6950b8..802361e 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,7 @@ request), `--retries N`, `--no-color`, and `--version`. ```bash sp investigate # one-shot triage: info + counts + classified failures sp run compare # which of these failures are new? +sp run report # ... same question, references resolved for you sp investigate --with-history # ... and whether each failure is new sp run summary # pass/fail summary for a run sp run failures # failing tests, each auto-classified diff --git a/sp_cli/commands/run.py b/sp_cli/commands/run.py index a2edccf..920637f 100644 --- a/sp_cli/commands/run.py +++ b/sp_cli/commands/run.py @@ -8,7 +8,7 @@ import click from sp_cli.client import ApiError -from sp_cli.compare import compare_runs, coverage_warnings +from sp_cli.compare import compare_runs, coverage_warnings, pick_references from sp_cli.constants import (ARTIFACT_TYPES, CANCEL_REASON_MIN_LENGTH, COMMIT_SHA_LENGTH, ERROR_GROUP_BY, ERROR_SEVERITIES, ERROR_TYPES, EXIT_TIMEOUT, @@ -16,7 +16,7 @@ LOG_CONTAINS_MAX_LENGTH, LOG_LEVELS, LOG_SOURCES, MAX_OFFSET, MAX_PAGE_LIMIT, MAX_REGRESSION_TEST_IDS, PLATFORMS, - PR_SCAN_DEFAULT, PR_SCAN_MAX, + PR_SCAN_DEFAULT, PR_SCAN_MAX, REPORT_LIST_LIMIT, RUN_PENDING_STATUSES, RUN_STATUSES, RUN_UNSUCCESSFUL_STATUSES, SAMPLE_STATUSES, WAIT_INTERVAL_DEFAULT, WAIT_INTERVAL_MAX, @@ -134,6 +134,144 @@ def _list_runs_for_pr(ctx: click.Context, params: Dict[str, Any], output, ctx.obj.get('color', False)) +@run.command('report') +@click.argument('run_id', type=int) +@click.option('--against', 'against', type=int, multiple=True, + help='Compare against these runs instead of resolving them (repeatable).') +@click.option('--branch', default='master', show_default=True, + help='Branch whose runs are used as references.') +@click.option('--scan', type=click.IntRange(1, MAX_PAGE_LIMIT), default=25, show_default=True, + help='How many recent branch runs to consider when resolving references.') +@click.pass_context +def run_report(ctx: click.Context, run_id: int, against: Tuple[int, ...], + branch: str, scan: int) -> None: + """Describe a run's failures against the approved output and against earlier runs. + + Pass and fail answer one question: did the output match the approved file. + Who made that true is a different question, and it is the one a reviewer + needs -- a test that has failed for a month says nothing about the change in + front of them. + + So the verdict is reported as-is, and every failure is then described + against earlier runs: the newest on the target branch, and the newest that + predates this run. A failure present in both is not this change's doing; one + that is new relative to the run before it is where to start reading. + + The second reference is a proxy for "where this branch was cut from", not + ancestry -- the API exposes no commit graph, so ordering by time is the best + available. Pass --against explicitly when you know the right baseline. + """ + client = ctx.obj['client'] + output = ctx.obj['output'] + try: + with Spinner(f'Building report for run {run_id}', output != 'json'): + run_detail = client.get(f'/runs/{run_id}') + run_samples = client.get_paginated(f'/runs/{run_id}/samples') + if against: + candidates = [client.get(f'/runs/{baseline_id}') for baseline_id in against] + else: + candidates = client.get_paginated( + '/runs', params={'branch': branch, 'platform': run_detail.get('platform')}, + max_items=scan) + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + + if against: + references = [{'label': 'the run you named', 'run': candidate} for candidate in candidates] + else: + references = pick_references(run_detail, candidates) + + failing = [row for row in run_samples if is_failure(row)] + report: Dict[str, Any] = { + 'run': {field: run_detail.get(field) for field in _COMPARE_RUN_FIELDS}, + 'verdict': { + 'against': 'the approved output', + 'tests_with_results': len(run_samples), + 'failing': len(failing), + }, + 'references': [], + } + + for reference in references: + try: + with Spinner(f"Comparing against run {reference['run']['run_id']}", output != 'json'): + baseline_samples = client.get_paginated( + f"/runs/{reference['run']['run_id']}/samples") + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + result = compare_runs(run_samples, baseline_samples) + report['references'].append({ + 'label': reference['label'], + 'run': {field: reference['run'].get(field) for field in _COMPARE_RUN_FIELDS}, + 'counts': result['counts'], + 'new': result['new'], + 'fixed': result['fixed'], + 'warnings': coverage_warnings(run_detail, reference['run'], result), + }) + + if output == 'json': + render(report, output, ctx.obj.get('color', False)) + return + _print_report(report, ctx.obj.get('color', False)) + + +def _print_report(report: Dict[str, Any], color: bool) -> None: + """ + Print a report as something a person reads top to bottom. + + :param report: The payload built by ``run report``. + :type report: Dict[str, Any] + :param color: Whether colour was requested. + :type color: bool + """ + run_info = report['run'] + verdict = report['verdict'] + click.echo(f"run {run_info['run_id']} {run_info['platform']} " + f"{(run_info.get('commit_sha') or '')[:8]} status={run_info.get('status')}") + click.echo(f" {verdict['failing']} of {verdict['tests_with_results']} tests " + f"do not match {verdict['against']}") + + if not report['references']: + click.echo('\n No earlier run to compare against, so nothing here says ' + 'whether this change caused them.') + return + + for reference in report['references']: + counts = reference['counts'] + reference_run = reference['run'] + click.echo(f"\n vs {reference['label']} — run {reference_run['run_id']} " + f"({(reference_run.get('commit_sha') or '')[:8]})") + tail = f" not_rerun {counts['not_rerun']}" if counts['not_rerun'] else '' + click.echo(f" new {counts['new']} changed {counts['changed']} " + f"still_failing {counts['still_failing']} fixed {counts['fixed']}{tail}") + for warning in reference['warnings']: + click.echo(f" note: {warning}", err=True) + if reference['new']: + # The counts are the answer; the list is evidence for it. A hundred + # rows under each reference buries the line the reader came for, so + # show enough to recognise the pattern and say what was held back. + shown = reference['new'][:REPORT_LIST_LIMIT] + render({'data': [{'regression_test_id': row.get('regression_test_id'), + 'sample_name': row.get('sample_name'), + 'code': row.get('code')} for row in shown]}, + 'table', color) + held_back = len(reference['new']) - len(shown) + if held_back: + click.echo(f" ... and {held_back} more (all of them in --output json)") + + nearest = report['references'][-1] + if nearest['counts']['new']: + click.echo(f"\n {nearest['counts']['new']} of {verdict['failing']} failures are new " + f"relative to {nearest['label']}. Those are this change's.") + elif verdict['failing']: + click.echo(f"\n None of the {verdict['failing']} failures are new relative to " + f"{nearest['label']}; they fail there too.") + else: + click.echo('\n Everything matched the approved output.') + + @run.command('create') @click.option('--commit', 'commit_sha', required=True, help=f'Full {COMMIT_SHA_LENGTH}-char commit SHA. Short SHAs are rejected.') diff --git a/sp_cli/compare.py b/sp_cli/compare.py index a81f9b0..fa186c0 100644 --- a/sp_cli/compare.py +++ b/sp_cli/compare.py @@ -155,3 +155,61 @@ def coverage_warnings(run: Dict[str, Any], baseline: Dict[str, Any], f"{result['counts']['not_rerun']} baseline failure(s) produced no result here; " 'they are reported as not_rerun rather than fixed.') return warnings + + +def pick_references(run: Dict[str, Any], + branch_runs: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Choose which earlier runs a run is worth being described against. + + Two references answer different questions. The newest run on the target + branch says whether a failure is broken where everyone else is working. The + newest one that predates this run is the closest thing to where the branch + was cut from, which is what separates "this change did it" from "it was + already like that". + + This is a *proxy* for ancestry, not ancestry: the API exposes no commit + graph, so a run that predates this one is assumed to precede it in history. + That holds for a branch cut from the target and stops holding for one cut + weeks ago and rebased since. The label says "before this run" rather than + "ancestor" so a reader is not told more than was checked. + + :param run: The run being reported on. + :type run: Dict[str, Any] + :param branch_runs: Candidate runs on the target branch, newest first. + :type branch_runs: List[Dict[str, Any]] + :return: References as {label, run}, nearest question first, deduplicated. + :rtype: List[Dict[str, Any]] + """ + usable = [] + for candidate in branch_runs: + if candidate.get('run_id') == run.get('run_id'): + continue + if candidate.get('platform') != run.get('platform'): + continue + # A run that never reached a verdict has nothing to say about this one. + if candidate.get('status') not in ('pass', 'fail'): + continue + usable.append(candidate) + if not usable: + return [] + + created = run.get('created_at') or '' + earlier = [] + if created: + for candidate in usable: + if (candidate.get('created_at') or '') < created: + earlier.append(candidate) + + chosen: List[Tuple[str, Dict[str, Any]]] = [('the newest run on the target branch', usable[0])] + if earlier: + chosen.append(('the newest run before this one', earlier[0])) + + references: List[Dict[str, Any]] = [] + seen: Set[int] = set() + for label, candidate in chosen: + if candidate['run_id'] in seen: + continue + seen.add(candidate['run_id']) + references.append({'label': label, 'run': candidate}) + return references diff --git a/sp_cli/constants.py b/sp_cli/constants.py index 6d540e4..a4218ab 100644 --- a/sp_cli/constants.py +++ b/sp_cli/constants.py @@ -45,6 +45,11 @@ #: do map to a code of their own (3 through 8) keep it. EXIT_WAIT_ABORTED = 10 +#: How many failing tests ``sp run report`` lists per reference in table mode. +#: The counts are the answer and the rows are evidence for it; JSON output is +#: never truncated, so nothing is lost to a script. +REPORT_LIST_LIMIT = 15 + #: ``sp run wait`` polling bounds, in seconds. WAIT_INTERVAL_DEFAULT = 30 WAIT_INTERVAL_MIN = 5 diff --git a/tests/test_cli.py b/tests/test_cli.py index 03e3156..3a2ed25 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1566,6 +1566,116 @@ def test_passing_the_default_explicitly_still_beats_a_saved_session(self): 'https://sampleplatform.ccextractor.org/api/v1') +class RunReportTests(unittest.TestCase): + """`sp run report` describes a run against the approved output and earlier runs.""" + + def setUp(self): + """Build a runner and the payloads every case is assembled from.""" + self.runner = CliRunner() + self.run = {'run_id': 9492, 'platform': 'linux', 'commit_sha': 'a' * 40, + 'branch': 'pull_request', 'pr_number': 2325, 'status': 'fail', + 'created_at': '2026-08-15T10:00:00Z'} + + @staticmethod + def sample(test_id, status, code='OUTPUT_DIFF'): + """ + Build one per-sample result row. + + :param test_id: Regression test id. + :type test_id: int + :param status: pass or fail. + :type status: str + :param code: Failure classification. + :type code: str + :return: A row shaped like /runs/{id}/samples returns. + :rtype: Dict[str, Any] + """ + return {'regression_test_id': test_id, 'sample_name': f'sample{test_id}', + 'status': status, 'code': code, 'exit_code': 0, 'expected_rc': 0} + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_a_failure_present_in_the_reference_is_not_reported_as_new(self, mock_get, mock_pages): + """The whole point: a failure the reference shares is not this change's. + + This is what the platform used to get wrong -- it reported 70 tests as + broken by a branch that had not touched them. + """ + mock_get.return_value = self.run + failing = [self.sample(i, 'fail') for i in range(1, 6)] + reference_run = {**self.run, 'run_id': 9484, 'branch': 'master', + 'created_at': '2026-08-14T10:00:00Z'} + mock_pages.side_effect = [failing, [reference_run], failing] + + result = self.runner.invoke(cli, ['run', 'report', '9492']) + payload = json.loads(result.stdout) + + self.assertEqual(payload['verdict']['failing'], 5) + self.assertEqual(payload['references'][0]['counts']['new'], 0) + self.assertEqual(payload['references'][0]['counts']['still_failing'], 5) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_a_failure_the_reference_passed_is_reported_as_new(self, mock_get, mock_pages): + """The complement: what the reference passed and this run failed is the finding.""" + mock_get.return_value = self.run + reference_run = {**self.run, 'run_id': 9484, 'branch': 'master', + 'created_at': '2026-08-14T10:00:00Z'} + mock_pages.side_effect = [[self.sample(1, 'fail')], [reference_run], + [self.sample(1, 'pass')]] + + payload = json.loads(self.runner.invoke(cli, ['run', 'report', '9492']).stdout) + + self.assertEqual(payload['references'][0]['counts']['new'], 1) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_no_reference_run_is_reported_rather_than_implied_to_be_clean(self, mock_get, mock_pages): + """Having nothing to compare against must not read as good news.""" + mock_get.return_value = self.run + mock_pages.side_effect = [[self.sample(1, 'fail')], []] + + result = self.runner.invoke(cli, ['run', 'report', '9492']) + payload = json.loads(result.stdout) + + self.assertEqual(payload['references'], []) + self.assertEqual(payload['verdict']['failing'], 1) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_explicit_baselines_are_used_instead_of_resolution(self, mock_get, mock_pages): + """--against names the comparison rather than guessing at it.""" + named = {**self.run, 'run_id': 9478, 'branch': 'master'} + mock_get.side_effect = [self.run, named] + mock_pages.side_effect = [[self.sample(1, 'fail')], [self.sample(1, 'fail')]] + + payload = json.loads( + self.runner.invoke(cli, ['run', 'report', '9492', '--against', '9478']).stdout) + + self.assertEqual(len(payload['references']), 1) + self.assertEqual(payload['references'][0]['run']['run_id'], 9478) + self.assertEqual(payload['references'][0]['label'], 'the run you named') + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_table_output_is_capped_but_json_is_complete(self, mock_get, mock_pages): + """Truncation is for reading; a script must still get everything.""" + mock_get.return_value = self.run + many = [self.sample(i, 'fail') for i in range(1, 40)] + reference_run = {**self.run, 'run_id': 9484, 'branch': 'master', + 'created_at': '2026-08-14T10:00:00Z'} + passing = [self.sample(i, 'pass') for i in range(1, 40)] + + mock_pages.side_effect = [many, [reference_run], passing] + table = self.runner.invoke(cli, ['-o', 'table', 'run', 'report', '9492']).stdout + + mock_pages.side_effect = [many, [reference_run], passing] + payload = json.loads(self.runner.invoke(cli, ['run', 'report', '9492']).stdout) + + self.assertIn('and 24 more', table) + self.assertEqual(len(payload['references'][0]['new']), 39) + + class RunWaitTests(unittest.TestCase): """Exercise `sp run wait`, with sleeping and the clock stubbed out.""" From f7a1f4e8ef51300c94f16333d7adeef99a869c68 Mon Sep 17 00:00:00 2001 From: Carlos Fernandez Date: Sun, 16 Aug 2026 22:48:48 -0700 Subject: [PATCH 2/2] feat(report): name every failing test and how each one stands A count is not something a reviewer can act on. "69 do not match" leaves them where they started: opening the web UI to find out which 69, and then working out which of those the reference already had. The report now names them, one row per failure, with a column per reference: test sample code vs 9494 vs 9484 11 Hoarders _AETVHD_2012... OUTPUT_DIFF fails there too NEW HERE 16 The Time That Remains... MISSING_OUT fails there too fails there too Which failures are this change's is then a column to read rather than a set difference to work out. The per-reference lists this replaces said the same thing three times over and still made the reader join them by eye. Codes come from classify_sample: the raw sample rows carry the ingredients -- exit codes, output states -- but not the verdict. --- sp_cli/commands/run.py | 70 ++++++++++++++++++++++++++++++++++-------- tests/test_cli.py | 47 ++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 12 deletions(-) diff --git a/sp_cli/commands/run.py b/sp_cli/commands/run.py index 920637f..ca15163 100644 --- a/sp_cli/commands/run.py +++ b/sp_cli/commands/run.py @@ -31,6 +31,18 @@ #: regressions first, missing evidence before good news. COMPARE_BUCKETS = ('new', 'changed', 'still_failing', 'not_rerun', 'fixed', 'no_baseline') +#: How a failure's standing against one reference reads in a table. Short +#: enough to keep a column narrow, and worded so a reader does not have to +#: remember which bucket name means what. +_STANDING_LABELS = { + 'new': 'NEW HERE', + 'changed': 'differs', + 'still_failing': 'fails there too', + 'fixed': 'fixed here', + 'not_rerun': 'not rerun', + 'no_baseline': 'never ran there', +} + #: Run fields identifying each side of a comparison. _COMPARE_RUN_FIELDS = ('run_id', 'platform', 'commit_sha', 'branch', 'pr_number', 'status') @@ -189,6 +201,14 @@ def run_report(ctx: click.Context, run_id: int, against: Tuple[int, ...], 'against': 'the approved output', 'tests_with_results': len(run_samples), 'failing': len(failing), + # Named, not just counted: "69 failed" is not something a reviewer + # can act on, and the whole point of the references below is to say + # something about each one of them. + # classify_sample supplies the code; the raw sample rows carry the + # ingredients for it (exit codes, output states) but not the verdict. + 'failures': [{'regression_test_id': row.get('regression_test_id'), + 'sample_name': row.get('sample_name'), + 'code': classify_sample(row).get('code')} for row in failing], }, 'references': [], } @@ -209,6 +229,7 @@ def run_report(ctx: click.Context, run_id: int, against: Tuple[int, ...], 'new': result['new'], 'fixed': result['fixed'], 'warnings': coverage_warnings(run_detail, reference['run'], result), + 'standing': _standing_by_test(result), }) if output == 'json': @@ -217,6 +238,24 @@ def run_report(ctx: click.Context, run_id: int, against: Tuple[int, ...], _print_report(report, ctx.obj.get('color', False)) +def _standing_by_test(result: Dict[str, Any]) -> Dict[int, str]: + """ + Map each regression test to how it stood against one reference. + + :param result: A ``compare_runs`` result. + :type result: Dict[str, Any] + :return: Regression test id mapped to its bucket name. + :rtype: Dict[int, str] + """ + standing: Dict[int, str] = {} + for bucket in ('new', 'changed', 'still_failing', 'fixed', 'not_rerun', 'no_baseline'): + for row in result.get(bucket, []): + test_id = row.get('regression_test_id') + if test_id is not None: + standing[test_id] = bucket + return standing + + def _print_report(report: Dict[str, Any], color: bool) -> None: """ Print a report as something a person reads top to bottom. @@ -248,18 +287,25 @@ def _print_report(report: Dict[str, Any], color: bool) -> None: f"still_failing {counts['still_failing']} fixed {counts['fixed']}{tail}") for warning in reference['warnings']: click.echo(f" note: {warning}", err=True) - if reference['new']: - # The counts are the answer; the list is evidence for it. A hundred - # rows under each reference buries the line the reader came for, so - # show enough to recognise the pattern and say what was held back. - shown = reference['new'][:REPORT_LIST_LIMIT] - render({'data': [{'regression_test_id': row.get('regression_test_id'), - 'sample_name': row.get('sample_name'), - 'code': row.get('code')} for row in shown]}, - 'table', color) - held_back = len(reference['new']) - len(shown) - if held_back: - click.echo(f" ... and {held_back} more (all of them in --output json)") + + failures = report['verdict']['failures'] + if failures: + click.echo(f"\n the {len(failures)} that do not match, and how each stands:") + shown = failures[:REPORT_LIST_LIMIT] + rows = [] + for failure in shown: + row = {'test': failure['regression_test_id'], + 'sample': failure['sample_name'], + 'code': failure['code']} + for reference in report['references']: + column = f"vs {reference['run']['run_id']}" + row[column] = _STANDING_LABELS.get( + reference['standing'].get(failure['regression_test_id']), 'no result') + rows.append(row) + render({'data': rows}, 'table', color) + held_back = len(failures) - len(shown) + if held_back: + click.echo(f" ... and {held_back} more (all of them in --output json)") nearest = report['references'][-1] if nearest['counts']['new']: diff --git a/tests/test_cli.py b/tests/test_cli.py index 3a2ed25..834f0b9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1675,6 +1675,53 @@ def test_table_output_is_capped_but_json_is_complete(self, mock_get, mock_pages) self.assertIn('and 24 more', table) self.assertEqual(len(payload['references'][0]['new']), 39) + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_every_failing_test_is_named_with_its_standing(self, mock_get, mock_pages): + """A count is not actionable: say which tests, and how each one stands. + + One row per failure with a column per reference is the whole answer -- + the reader can see at a glance which failures are theirs and which the + reference already had. + """ + mock_get.return_value = self.run + reference_run = {**self.run, 'run_id': 9484, 'branch': 'master', + 'created_at': '2026-08-14T10:00:00Z'} + mock_pages.side_effect = [ + [self.sample(1, 'fail'), self.sample(2, 'fail'), self.sample(3, 'pass')], + [reference_run], + [self.sample(1, 'pass'), self.sample(2, 'fail'), self.sample(3, 'pass')], + ] + + payload = json.loads(self.runner.invoke(cli, ['run', 'report', '9492']).stdout) + named = {f['regression_test_id']: f for f in payload['verdict']['failures']} + standing = payload['references'][0]['standing'] + + self.assertEqual(sorted(named), [1, 2]) + self.assertTrue(all(f['code'] for f in payload['verdict']['failures'])) + self.assertEqual(standing['1'], 'new') + self.assertEqual(standing['2'], 'still_failing') + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_table_names_the_failures_and_their_standing(self, mock_get, mock_pages): + """The table a human reads must carry the same answer as the JSON.""" + mock_get.return_value = self.run + reference_run = {**self.run, 'run_id': 9484, 'branch': 'master', + 'created_at': '2026-08-14T10:00:00Z'} + mock_pages.side_effect = [ + [self.sample(1, 'fail'), self.sample(2, 'fail')], + [reference_run], + [self.sample(1, 'pass'), self.sample(2, 'fail')], + ] + + table = self.runner.invoke(cli, ['-o', 'table', 'run', 'report', '9492']).stdout + + self.assertIn('sample1', table) + self.assertIn('sample2', table) + self.assertIn('NEW HERE', table) + self.assertIn('fails there too', table) + class RunWaitTests(unittest.TestCase): """Exercise `sp run wait`, with sleeping and the clock stubbed out."""