Skip to content

Commit 241f999

Browse files
feat: add process-test-results command (#424)
* feat: add process-test-results command this command locally processes test result files and makes a call to the GH API to create a comment This command should be run in Github Actions, it expects the provider-token option to contain the contents of the GITHUB_TOKEN env var. * deps: update requirements.txt * fix: use typing List instead of list Signed-off-by: joseph-sentry <joseph.sawaya@sentry.io>
1 parent 3abcc36 commit 241f999

8 files changed

Lines changed: 464 additions & 6 deletions

File tree

.github/workflows/ci.yml

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,3 +132,38 @@ jobs:
132132
- name: Upload smart-labels
133133
run: |
134134
codecovcli --codecov-yml-path=codecov.yml do-upload --plugin pycoverage --plugin compress-pycoverage --fail-on-error -t ${{ secrets.CODECOV_TOKEN }} --flag smart-labels
135+
136+
test-process-test-results-cmd:
137+
runs-on: ubuntu-latest
138+
permissions:
139+
pull-requests: write
140+
strategy:
141+
fail-fast: false
142+
matrix:
143+
include:
144+
- python-version: "3.11"
145+
- python-version: "3.10"
146+
- python-version: "3.9"
147+
- python-version: "3.8"
148+
steps:
149+
- uses: actions/checkout@v4
150+
with:
151+
submodules: true
152+
fetch-depth: 2
153+
- name: Set up Python ${{matrix.python-version}}
154+
uses: actions/setup-python@v3
155+
with:
156+
python-version: "${{matrix.python-version}}"
157+
- name: Install dependencies
158+
run: |
159+
python -m pip install --upgrade pip
160+
pip install -r requirements.txt
161+
python setup.py develop
162+
pip install -r tests/requirements.txt
163+
- name: Test with pytest
164+
run: |
165+
pytest --cov --junitxml=junit.xml
166+
- name: Dogfooding codecov-cli
167+
if: ${{ !cancelled() }}
168+
run: |
169+
codecovcli process-test-results --provider-token ${{ secrets.GITHUB_TOKEN }}
Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
import logging
2+
import os
3+
import pathlib
4+
from dataclasses import dataclass
5+
from typing import List
6+
7+
import click
8+
from test_results_parser import (
9+
Outcome,
10+
ParserError,
11+
Testrun,
12+
build_message,
13+
parse_junit_xml,
14+
)
15+
16+
from codecov_cli.helpers.request import (
17+
log_warnings_and_errors_if_any,
18+
send_post_request,
19+
)
20+
from codecov_cli.services.upload.file_finder import select_file_finder
21+
22+
logger = logging.getLogger("codecovcli")
23+
24+
25+
_process_test_results_options = [
26+
click.option(
27+
"-s",
28+
"--dir",
29+
"--files-search-root-folder",
30+
"dir",
31+
help="Folder where to search for test results files",
32+
type=click.Path(path_type=pathlib.Path),
33+
default=pathlib.Path.cwd,
34+
show_default="Current Working Directory",
35+
),
36+
click.option(
37+
"-f",
38+
"--file",
39+
"--files-search-direct-file",
40+
"files",
41+
help="Explicit files to upload. These will be added to the test results files to be processed. If you wish to only process the specified files, please consider using --disable-search to disable processing other files.",
42+
type=click.Path(path_type=pathlib.Path),
43+
multiple=True,
44+
default=[],
45+
),
46+
click.option(
47+
"--exclude",
48+
"--files-search-exclude-folder",
49+
"exclude_folders",
50+
help="Folders to exclude from search",
51+
type=click.Path(path_type=pathlib.Path),
52+
multiple=True,
53+
default=[],
54+
),
55+
click.option(
56+
"--disable-search",
57+
help="Disable search for coverage files. This is helpful when specifying what files you want to upload with the --file option.",
58+
is_flag=True,
59+
default=False,
60+
),
61+
click.option(
62+
"--provider-token",
63+
help="Token used to make calls to Repo provider API",
64+
type=str,
65+
default=None,
66+
),
67+
]
68+
69+
70+
def process_test_results_options(func):
71+
for option in reversed(_process_test_results_options):
72+
func = option(func)
73+
return func
74+
75+
76+
@dataclass
77+
class TestResultsNotificationPayload:
78+
failures: List[Testrun]
79+
failed: int = 0
80+
passed: int = 0
81+
skipped: int = 0
82+
83+
84+
@click.command()
85+
@process_test_results_options
86+
def process_test_results(
87+
dir=None, files=None, exclude_folders=None, disable_search=None, provider_token=None
88+
):
89+
if provider_token is None:
90+
raise click.ClickException(
91+
"Provider token was not provided. Make sure to pass --provider-token option with the contents of the GITHUB_TOKEN secret, so we can make a comment."
92+
)
93+
94+
summary_file_path = os.getenv("GITHUB_STEP_SUMMARY")
95+
if summary_file_path is None:
96+
raise click.ClickException(
97+
"Error getting step summary file path from environment. Can't find GITHUB_STEP_SUMMARY environment variable."
98+
)
99+
100+
slug = os.getenv("GITHUB_REPOSITORY")
101+
if slug is None:
102+
raise click.ClickException(
103+
"Error getting repo slug from environment. Can't find GITHUB_REPOSITORY environment variable."
104+
)
105+
106+
ref = os.getenv("GITHUB_REF")
107+
if ref is None or "pull" not in ref:
108+
raise click.ClickException(
109+
"Error getting PR number from environment. Can't find GITHUB_REF environment variable."
110+
)
111+
112+
file_finder = select_file_finder(
113+
dir, exclude_folders, files, disable_search, report_type="test_results"
114+
)
115+
116+
upload_collection_results = file_finder.find_files()
117+
if len(upload_collection_results) == 0:
118+
raise click.ClickException(
119+
"No JUnit XML files were found. Make sure to specify them using the --file option."
120+
)
121+
122+
payload = generate_message_payload(upload_collection_results)
123+
124+
message = build_message(payload)
125+
126+
# write to step summary file
127+
with open(summary_file_path, "w") as f:
128+
f.write(message)
129+
130+
# GITHUB_REF is documented here: https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
131+
pr_number = ref.split("/")[2]
132+
133+
create_github_comment(provider_token, slug, pr_number, message)
134+
135+
136+
def create_github_comment(token, repo_slug, pr_number, message):
137+
url = f"https://api.github.com/repos/{repo_slug}/issues/{pr_number}/comments"
138+
139+
headers = {
140+
"Accept": "application/vnd.github+json",
141+
"Authorization": f"Bearer {token}",
142+
"X-GitHub-Api-Version": "2022-11-28",
143+
}
144+
logger.info("Posting github comment")
145+
146+
log_warnings_and_errors_if_any(
147+
send_post_request(url=url, data={"body": message}, headers=headers),
148+
"Posting test results comment",
149+
)
150+
151+
152+
def generate_message_payload(upload_collection_results):
153+
payload = TestResultsNotificationPayload(failures=[])
154+
155+
for result in upload_collection_results:
156+
testruns = []
157+
try:
158+
logger.info(f"Parsing {result.get_filename()}")
159+
testruns = parse_junit_xml(result.get_content())
160+
for testrun in testruns:
161+
if (
162+
testrun.outcome == Outcome.Failure
163+
or testrun.outcome == Outcome.Error
164+
):
165+
payload.failed += 1
166+
payload.failures.append(testrun)
167+
elif testrun.outcome == Outcome.Skip:
168+
payload.skipped += 1
169+
else:
170+
payload.passed += 1
171+
except ParserError as err:
172+
raise click.ClickException(
173+
f"Error parsing {str(result.get_filename(), 'utf8')} with error: {err}"
174+
)
175+
return payload

codecov_cli/main.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from codecov_cli.commands.empty_upload import empty_upload
1212
from codecov_cli.commands.get_report_results import get_report_results
1313
from codecov_cli.commands.labelanalysis import label_analysis
14+
from codecov_cli.commands.process_test_results import process_test_results
1415
from codecov_cli.commands.report import create_report
1516
from codecov_cli.commands.send_notifications import send_notifications
1617
from codecov_cli.commands.staticanalysis import static_analysis
@@ -71,6 +72,7 @@ def cli(
7172
cli.add_command(empty_upload)
7273
cli.add_command(upload_process)
7374
cli.add_command(send_notifications)
75+
cli.add_command(process_test_results)
7476

7577

7678
def run():

requirements.txt

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# This file is autogenerated by pip-compile with Python 3.10
2+
# This file is autogenerated by pip-compile with Python 3.11
33
# by the following command:
44
#
55
# pip-compile setup.py
@@ -15,8 +15,6 @@ charset-normalizer==3.3.0
1515
# via requests
1616
click==8.1.7
1717
# via codecov-cli (setup.py)
18-
exceptiongroup==1.1.3
19-
# via anyio
2018
h11==0.14.0
2119
# via httpcore
2220
httpcore==0.16.3
@@ -32,19 +30,21 @@ ijson==3.2.3
3230
# via codecov-cli (setup.py)
3331
pyyaml==6.0.1
3432
# via codecov-cli (setup.py)
33+
regex==2023.12.25
34+
# via codecov-cli (setup.py)
3535
requests==2.31.0
3636
# via responses
3737
responses==0.21.0
3838
# via codecov-cli (setup.py)
3939
rfc3986[idna2008]==1.5.0
40-
# via
41-
# httpx
42-
# rfc3986
40+
# via httpx
4341
sniffio==1.3.0
4442
# via
4543
# anyio
4644
# httpcore
4745
# httpx
46+
test-results-parser==0.1.0
47+
# via codecov-cli (setup.py)
4848
tree-sitter==0.20.2
4949
# via codecov-cli (setup.py)
5050
urllib3==2.0.6

samples/junit.xml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<testsuites>
3+
<testsuite name="pytest" errors="0" failures="1" skipped="0" tests="4" time="0.052"
4+
timestamp="2023-11-06T11:17:04.011072" hostname="VFHNWJDWH9.local">
5+
<testcase classname="api.temp.calculator.test_calculator" name="test_add" time="0.001" />
6+
<testcase classname="api.temp.calculator.test_calculator" name="test_subtract" time="0.001" />
7+
<testcase classname="api.temp.calculator.test_calculator" name="test_multiply" time="0.000" />
8+
<testcase classname="api.temp.calculator.test_calculator" name="test_divide" time="0.001">
9+
<failure
10+
message="assert 1.0 == 0.5&#10; + where 1.0 = &lt;function Calculator.divide at 0x104c9eb90&gt;(1, 2)&#10; + where &lt;function Calculator.divide at 0x104c9eb90&gt; = Calculator.divide">def
11+
test_divide():
12+
&gt; assert Calculator.divide(1, 2) == 0.5
13+
E assert 1.0 == 0.5
14+
E + where 1.0 = &lt;function Calculator.divide at 0x104c9eb90&gt;(1, 2)
15+
E + where &lt;function Calculator.divide at 0x104c9eb90&gt; = Calculator.divide
16+
api/temp/calculator/test_calculator.py:30: AssertionError</failure>
17+
</testcase>
18+
</testsuite>
19+
</testsuites>

setup.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
"pyyaml==6.*",
2525
"responses==0.21.*",
2626
"tree-sitter==0.20.*",
27+
"test-results-parser==0.1.*",
28+
"regex",
2729
],
2830
entry_points={
2931
"console_scripts": [

0 commit comments

Comments
 (0)