Skip to content

Commit 5481b9e

Browse files
MichaelYochpazMichaelYochpaz
committed
Add benchmarks framework
Add performance benchmarking infrastructure for fromager including GitHub Actions workflows for nightly and on-demand runs. Signed-off-by: Michael Yochpaz <myochpaz@redhat.com>
1 parent f6e78fb commit 5481b9e

14 files changed

Lines changed: 1191 additions & 0 deletions

‎.github/workflows/benchmark-backfill.yml‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
name: Benchmarks (Backfill)
2+
3+
# This workflow runs up-to-date benchmarks on historical commits to populate
4+
# CodSpeed with baseline data. It uses "Runtime Dependency Injection":
5+
#
6+
# 1. Checkout the historical commit (code under test)
7+
# 2. Copy benchmarks/ from source branch (modern test harness)
8+
# 3. Install project using historical pyproject.toml (correct runtime deps)
9+
# 4. Extract benchmark deps from source branch's pyproject.toml (single source of truth)
10+
# 5. Run pytest directly (bypasses missing Hatch env in old commits)
11+
#
12+
# See `benchmarks/README.md` for more information.
13+
14+
on:
15+
workflow_dispatch:
16+
inputs:
17+
from_commit:
18+
description: 'Start commit SHA (older). Max 200 commits (or 128 with integration).'
19+
required: true
20+
type: string
21+
to_commit:
22+
description: 'End commit SHA (newer). Max 200 commits (or 128 with integration).'
23+
required: false
24+
default: 'HEAD'
25+
type: string
26+
benchmark_source:
27+
description: 'Branch to copy benchmarks from'
28+
required: false
29+
default: 'main'
30+
type: string
31+
benchmark_set:
32+
description: 'Benchmark set to run'
33+
required: false
34+
default: 'fast'
35+
type: choice
36+
options:
37+
- fast
38+
- full
39+
include_integration:
40+
description: 'Include integration benchmarks (uses Macro Runners)'
41+
required: false
42+
default: false
43+
type: boolean
44+
45+
permissions:
46+
contents: read
47+
id-token: write
48+
49+
jobs:
50+
prepare:
51+
runs-on: ubuntu-latest
52+
outputs:
53+
commits: ${{ steps.get-commits.outputs.commits }}
54+
steps:
55+
- uses: actions/checkout@v4
56+
with:
57+
fetch-depth: 0
58+
59+
- name: Get commits between range
60+
id: get-commits
61+
run: |
62+
FROM="${{ inputs.from_commit }}"
63+
TO="${{ inputs.to_commit }}"
64+
65+
# GitHub Actions has a hard limit of 256 jobs per matrix.
66+
# When integration benchmarks are enabled, both component and integration
67+
# jobs run on each commit (2 jobs per commit), so we halve the limit.
68+
if [ "${{ inputs.include_integration }}" = "true" ]; then
69+
MAX_COMMITS=128 # 128 commits × 2 jobs = 256 jobs max
70+
else
71+
MAX_COMMITS=200 # Leave room for future matrix expansion
72+
fi
73+
74+
# Get the most recent commits in the range
75+
# rev-list outputs newest first, head takes the N newest, tac reverses to chronological order
76+
COMMITS=$(git rev-list "$FROM^..$TO" | head -$MAX_COMMITS | tac)
77+
78+
TOTAL=$(git rev-list "$FROM^..$TO" | wc -l | tr -d ' ')
79+
SELECTED=$(echo "$COMMITS" | wc -l | tr -d ' ')
80+
81+
if [ "$TOTAL" -gt "$MAX_COMMITS" ]; then
82+
echo "::warning::Range contains $TOTAL commits, but only the $MAX_COMMITS most recent will be benchmarked (GitHub Actions limit is 256 jobs per matrix)."
83+
fi
84+
85+
# Convert to JSON array
86+
JSON_COMMITS=$(echo "$COMMITS" | jq -R -s -c 'split("\n") | map(select(length > 0))')
87+
echo "commits=$JSON_COMMITS" >> $GITHUB_OUTPUT
88+
echo "Will benchmark $SELECTED commits (out of $TOTAL in range):"
89+
echo "$COMMITS"
90+
91+
# Component benchmarks: CPU-bound, pure Python operations
92+
# Uses CPU simulation on standard GitHub runners
93+
backfill-component:
94+
needs: prepare
95+
runs-on: ubuntu-latest
96+
timeout-minutes: 60
97+
strategy:
98+
fail-fast: false
99+
max-parallel: 5 # Throttle to avoid overwhelming CodSpeed ingestion
100+
matrix:
101+
commit: ${{ fromJson(needs.prepare.outputs.commits) }}
102+
103+
steps:
104+
- uses: actions/checkout@v4
105+
with:
106+
ref: ${{ matrix.commit }}
107+
fetch-depth: 0
108+
109+
- name: Fetch benchmark assets from source branch
110+
run: |
111+
git fetch origin ${{ inputs.benchmark_source }}
112+
rm -rf benchmarks/
113+
git restore --source=origin/${{ inputs.benchmark_source }} --worktree benchmarks/
114+
git show origin/${{ inputs.benchmark_source }}:pyproject.toml > /tmp/source_pyproject.toml
115+
116+
- name: Set up Python
117+
uses: actions/setup-python@v5
118+
with:
119+
python-version: "3.11"
120+
121+
- name: Install uv
122+
uses: astral-sh/setup-uv@v4
123+
124+
- name: Install project (uses historical pyproject.toml)
125+
run: uv pip install -e . --system
126+
127+
- name: Install benchmark dependencies (from source branch)
128+
run: |
129+
python benchmarks/scripts/extract_deps.py /tmp/source_pyproject.toml \
130+
| uv pip install -r - --system
131+
132+
- name: Run component benchmarks with CodSpeed
133+
uses: CodSpeedHQ/action@v4
134+
with:
135+
mode: simulation
136+
run: |
137+
if [ "${{ inputs.benchmark_set }}" = "fast" ]; then
138+
pytest benchmarks/ --codspeed -m "not slow and not integration"
139+
else
140+
pytest benchmarks/ --codspeed -m "not integration"
141+
fi
142+
143+
# Integration benchmarks: I/O-bound operations with network and file access
144+
# Uses walltime on Macro Runners - only runs if include_integration is true
145+
backfill-integration:
146+
needs: prepare
147+
if: ${{ inputs.include_integration }}
148+
runs-on: codspeed-macro
149+
timeout-minutes: 60
150+
strategy:
151+
fail-fast: false
152+
max-parallel: 2 # Lower parallelism to conserve 600 min/month Macro Runner quota
153+
matrix:
154+
commit: ${{ fromJson(needs.prepare.outputs.commits) }}
155+
156+
steps:
157+
- uses: actions/checkout@v4
158+
with:
159+
ref: ${{ matrix.commit }}
160+
fetch-depth: 0
161+
162+
- name: Fetch benchmark assets from source branch
163+
run: |
164+
git fetch origin ${{ inputs.benchmark_source }}
165+
rm -rf benchmarks/
166+
git restore --source=origin/${{ inputs.benchmark_source }} --worktree benchmarks/
167+
git show origin/${{ inputs.benchmark_source }}:pyproject.toml > /tmp/source_pyproject.toml
168+
169+
- name: Set up Python
170+
uses: actions/setup-python@v5
171+
with:
172+
python-version: "3.11"
173+
174+
- name: Install uv
175+
uses: astral-sh/setup-uv@v4
176+
177+
- name: Install project (uses historical pyproject.toml)
178+
run: uv pip install -e . --system
179+
180+
- name: Install benchmark dependencies (from source branch)
181+
run: |
182+
python benchmarks/scripts/extract_deps.py /tmp/source_pyproject.toml \
183+
| uv pip install -r - --system
184+
185+
- name: Run integration benchmarks with CodSpeed (walltime)
186+
uses: CodSpeedHQ/action@v4
187+
with:
188+
mode: walltime
189+
run: |
190+
if [ "${{ inputs.benchmark_set }}" = "fast" ]; then
191+
pytest benchmarks/ --codspeed -m "integration and not slow"
192+
else
193+
pytest benchmarks/ --codspeed -m "integration"
194+
fi

‎.github/workflows/benchmarks-integration.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: Benchmarks (Integration)
2+
3+
# Runs integration benchmarks on PRs and main using walltime mode on CodSpeed
4+
# Macro Runners. Walltime accurately measures I/O, network, and system calls.
5+
# See benchmarks/README.md for details.
6+
7+
on:
8+
pull_request:
9+
push:
10+
branches: [main]
11+
12+
permissions:
13+
contents: read
14+
id-token: write
15+
16+
jobs:
17+
benchmark-integration:
18+
runs-on: codspeed-macro
19+
timeout-minutes: 30
20+
21+
steps:
22+
- uses: actions/checkout@v4
23+
with:
24+
fetch-depth: 0
25+
26+
- name: Set up Python
27+
uses: actions/setup-python@v5
28+
with:
29+
python-version: "3.11"
30+
31+
- name: Install hatch
32+
run: pip install hatch
33+
34+
- name: Run integration benchmarks with CodSpeed (walltime)
35+
uses: CodSpeedHQ/action@v4
36+
with:
37+
mode: walltime
38+
run: hatch run benchmark:run --codspeed -m "integration and not slow"
39+
40+
- name: Generate benchmark JSON (fallback)
41+
if: always()
42+
run: |
43+
hatch run benchmark:run \
44+
--benchmark-only \
45+
--benchmark-json=benchmark-results-integration.json \
46+
-m "integration and not slow" || true
47+
48+
- name: Upload benchmark results
49+
uses: actions/upload-artifact@v4
50+
if: always()
51+
with:
52+
name: benchmark-results-integration
53+
path: benchmark-results-integration.json
54+
retention-days: 30

‎.github/workflows/benchmarks-nightly.yml‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
name: Benchmarks (Nightly)
2+
3+
# Runs full benchmark suite nightly or on PRs with 'run-benchmarks' label.
4+
# Component benchmarks use CPU simulation, integration benchmarks use walltime.
5+
# Skips if no commits in 24 hours. See benchmarks/README.md for details.
6+
7+
on:
8+
schedule:
9+
- cron: "0 2 * * *" # 2 AM UTC daily
10+
workflow_dispatch: # Allow manual trigger
11+
pull_request:
12+
types: [labeled]
13+
14+
permissions:
15+
contents: read
16+
id-token: write
17+
18+
jobs:
19+
check-changes:
20+
runs-on: ubuntu-latest
21+
outputs:
22+
should_run: ${{ steps.check.outputs.should_run }}
23+
steps:
24+
- uses: actions/checkout@v4
25+
with:
26+
fetch-depth: 2
27+
28+
- name: Check if should run
29+
id: check
30+
run: |
31+
# Always run for label triggers and manual dispatch
32+
if [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
33+
echo "should_run=true" >> $GITHUB_OUTPUT
34+
exit 0
35+
fi
36+
37+
# For scheduled runs, skip if HEAD hasn't changed in 24 hours
38+
LAST_COMMIT_TIME=$(git log -1 --format=%ct)
39+
NOW=$(date +%s)
40+
HOURS_AGO=$(( (NOW - LAST_COMMIT_TIME) / 3600 ))
41+
42+
if [ "$HOURS_AGO" -gt 24 ]; then
43+
echo "No commits in the last 24 hours, skipping nightly benchmark"
44+
echo "should_run=false" >> $GITHUB_OUTPUT
45+
else
46+
echo "should_run=true" >> $GITHUB_OUTPUT
47+
fi
48+
49+
# Component benchmarks: CPU-bound, pure Python operations
50+
# Uses CPU simulation for deterministic, hardware-independent measurements
51+
component-benchmarks:
52+
needs: check-changes
53+
if: |
54+
needs.check-changes.outputs.should_run == 'true' &&
55+
(github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks'))
56+
runs-on: ubuntu-latest
57+
timeout-minutes: 60
58+
59+
steps:
60+
- uses: actions/checkout@v4
61+
with:
62+
fetch-depth: 0
63+
64+
- name: Set up Python
65+
uses: actions/setup-python@v5
66+
with:
67+
python-version: "3.11"
68+
69+
- name: Install hatch
70+
run: pip install hatch
71+
72+
- name: Run component benchmarks with CodSpeed
73+
uses: CodSpeedHQ/action@v4
74+
with:
75+
mode: simulation
76+
run: hatch run benchmark:run --codspeed -m "not integration"
77+
78+
- name: Generate benchmark JSON (fallback)
79+
if: always()
80+
run: |
81+
hatch run benchmark:run \
82+
--benchmark-only \
83+
--benchmark-json=benchmark-results-component.json \
84+
-m "not integration" || true
85+
86+
- name: Upload benchmark results
87+
uses: actions/upload-artifact@v4
88+
if: always()
89+
with:
90+
name: benchmark-results-nightly-component
91+
path: benchmark-results-component.json
92+
retention-days: 90
93+
94+
# Integration benchmarks: I/O-bound operations with network and file access
95+
# Uses walltime on Macro Runners for accurate real-world measurements
96+
integration-benchmarks:
97+
needs: check-changes
98+
if: |
99+
needs.check-changes.outputs.should_run == 'true' &&
100+
(github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks'))
101+
runs-on: codspeed-macro
102+
timeout-minutes: 60
103+
104+
steps:
105+
- uses: actions/checkout@v4
106+
with:
107+
fetch-depth: 0
108+
109+
- name: Set up Python
110+
uses: actions/setup-python@v5
111+
with:
112+
python-version: "3.11"
113+
114+
- name: Install hatch
115+
run: pip install hatch
116+
117+
- name: Run integration benchmarks with CodSpeed (walltime)
118+
uses: CodSpeedHQ/action@v4
119+
with:
120+
mode: walltime
121+
run: hatch run benchmark:run --codspeed -m "integration"
122+
123+
- name: Generate benchmark JSON (fallback)
124+
if: always()
125+
run: |
126+
hatch run benchmark:run \
127+
--benchmark-only \
128+
--benchmark-json=benchmark-results-integration.json \
129+
-m "integration" || true
130+
131+
- name: Upload benchmark results
132+
uses: actions/upload-artifact@v4
133+
if: always()
134+
with:
135+
name: benchmark-results-nightly-integration
136+
path: benchmark-results-integration.json
137+
retention-days: 90

0 commit comments

Comments
 (0)