Skip to content

Commit 2bfd5fb

Browse files
MichaelYochpazMichaelYochpaz
committed
Add benchmarks framework
Add performance benchmarking infrastructure for fromager including GitHub Actions workflows for nightly and on-demand runs. Signed-off-by: Michael Yochpaz <myochpaz@redhat.com>
1 parent f6e78fb commit 2bfd5fb

File tree

14 files changed

+1197
-0
lines changed

14 files changed

+1197
-0
lines changed

‎.github/workflows/benchmark-backfill.yml‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
name: Benchmarks (Backfill)
2+
3+
# This workflow runs up-to-date benchmarks on historical commits to populate
4+
# CodSpeed with baseline data. It uses "Runtime Dependency Injection":
5+
#
6+
# 1. Checkout the historical commit (code under test)
7+
# 2. Copy benchmarks/ from source branch (modern test harness)
8+
# 3. Install project using historical pyproject.toml (correct runtime deps)
9+
# 4. Extract benchmark deps from source branch's pyproject.toml (single source of truth)
10+
# 5. Run pytest directly (bypasses missing Hatch env in old commits)
11+
#
12+
# See `benchmarks/README.md` for more information.
13+
14+
on:
15+
workflow_dispatch:
16+
inputs:
17+
from_commit:
18+
description: 'Start commit SHA (older). Max 200 commits (or 128 with integration).'
19+
required: true
20+
type: string
21+
to_commit:
22+
description: 'End commit SHA (newer). Max 200 commits (or 128 with integration).'
23+
required: false
24+
default: 'HEAD'
25+
type: string
26+
benchmark_source:
27+
description: 'Branch to copy benchmarks from'
28+
required: false
29+
default: 'main'
30+
type: string
31+
benchmark_set:
32+
description: 'Benchmark set to run'
33+
required: false
34+
default: 'fast'
35+
type: choice
36+
options:
37+
- fast
38+
- full
39+
include_integration:
40+
description: 'Include integration benchmarks (uses Macro Runners)'
41+
required: false
42+
default: false
43+
type: boolean
44+
45+
permissions:
46+
contents: read
47+
id-token: write
48+
49+
jobs:
50+
prepare:
51+
runs-on: ubuntu-latest
52+
outputs:
53+
commits: ${{ steps.get-commits.outputs.commits }}
54+
steps:
55+
- uses: actions/checkout@v4
56+
with:
57+
fetch-depth: 0
58+
59+
- name: Get commits between range
60+
id: get-commits
61+
run: |
62+
FROM="${{ inputs.from_commit }}"
63+
TO="${{ inputs.to_commit }}"
64+
65+
# GitHub Actions has a hard limit of 256 jobs per matrix.
66+
# When integration benchmarks are enabled, both component and integration
67+
# jobs run on each commit (2 jobs per commit), so we halve the limit.
68+
if [ "${{ inputs.include_integration }}" = "true" ]; then
69+
MAX_COMMITS=128 # 128 commits × 2 jobs = 256 jobs max
70+
else
71+
MAX_COMMITS=200 # Leave room for future matrix expansion
72+
fi
73+
74+
# Get the most recent commits in the range
75+
# rev-list outputs newest first, head takes the N newest, tac reverses to chronological order
76+
COMMITS=$(git rev-list "$FROM^..$TO" | head -$MAX_COMMITS | tac)
77+
78+
TOTAL=$(git rev-list "$FROM^..$TO" | wc -l | tr -d ' ')
79+
SELECTED=$(echo "$COMMITS" | wc -l | tr -d ' ')
80+
81+
if [ "$TOTAL" -gt "$MAX_COMMITS" ]; then
82+
echo "::warning::Range contains $TOTAL commits, but only the $MAX_COMMITS most recent will be benchmarked (GitHub Actions limit is 256 jobs per matrix)."
83+
fi
84+
85+
# Convert to JSON array
86+
JSON_COMMITS=$(echo "$COMMITS" | jq -R -s -c 'split("\n") | map(select(length > 0))')
87+
echo "commits=$JSON_COMMITS" >> $GITHUB_OUTPUT
88+
echo "Will benchmark $SELECTED commits (out of $TOTAL in range):"
89+
echo "$COMMITS"
90+
91+
# Component benchmarks: CPU-bound, pure Python operations
92+
# Uses CPU simulation on standard GitHub runners
93+
backfill-component:
94+
needs: prepare
95+
runs-on: ubuntu-latest
96+
timeout-minutes: 60
97+
strategy:
98+
fail-fast: false
99+
max-parallel: 5 # Throttle to avoid overwhelming CodSpeed ingestion
100+
matrix:
101+
commit: ${{ fromJson(needs.prepare.outputs.commits) }}
102+
103+
steps:
104+
- uses: actions/checkout@v4
105+
with:
106+
ref: ${{ matrix.commit }}
107+
fetch-depth: 0
108+
109+
- name: Fetch benchmark assets from source branch
110+
run: |
111+
git fetch origin ${{ inputs.benchmark_source }}
112+
rm -rf benchmarks/
113+
git restore --source=origin/${{ inputs.benchmark_source }} --worktree benchmarks/
114+
git show origin/${{ inputs.benchmark_source }}:pyproject.toml > /tmp/source_pyproject.toml
115+
116+
- name: Set up Python
117+
uses: actions/setup-python@v5
118+
with:
119+
python-version: "3.11"
120+
121+
- name: Install uv
122+
uses: astral-sh/setup-uv@v4
123+
124+
- name: Install project (uses historical pyproject.toml)
125+
run: uv pip install -e . --system
126+
127+
- name: Install benchmark dependencies (from source branch)
128+
run: |
129+
python benchmarks/scripts/extract_deps.py /tmp/source_pyproject.toml \
130+
| uv pip install -r - --system
131+
132+
- name: Run component benchmarks with CodSpeed
133+
uses: CodSpeedHQ/action@v4
134+
with:
135+
mode: simulation
136+
run: |
137+
if [ "${{ inputs.benchmark_set }}" = "fast" ]; then
138+
pytest benchmarks/ --codspeed -m "not slow and not integration"
139+
else
140+
pytest benchmarks/ --codspeed -m "not integration"
141+
fi
142+
143+
# Integration benchmarks: I/O-bound operations with network and file access
144+
# Uses walltime on Macro Runners - only runs if include_integration is true
145+
backfill-integration:
146+
needs: prepare
147+
if: ${{ inputs.include_integration }}
148+
runs-on: codspeed-macro
149+
timeout-minutes: 60
150+
strategy:
151+
fail-fast: false
152+
max-parallel: 2 # Lower parallelism to conserve 600 min/month Macro Runner quota
153+
matrix:
154+
commit: ${{ fromJson(needs.prepare.outputs.commits) }}
155+
156+
steps:
157+
- uses: actions/checkout@v4
158+
with:
159+
ref: ${{ matrix.commit }}
160+
fetch-depth: 0
161+
162+
- name: Fetch benchmark assets from source branch
163+
run: |
164+
git fetch origin ${{ inputs.benchmark_source }}
165+
rm -rf benchmarks/
166+
git restore --source=origin/${{ inputs.benchmark_source }} --worktree benchmarks/
167+
git show origin/${{ inputs.benchmark_source }}:pyproject.toml > /tmp/source_pyproject.toml
168+
169+
- name: Set up Python
170+
uses: actions/setup-python@v5
171+
with:
172+
python-version: "3.11"
173+
174+
- name: Install uv
175+
uses: astral-sh/setup-uv@v4
176+
177+
- name: Install project (uses historical pyproject.toml)
178+
run: uv pip install -e . --system
179+
180+
- name: Install benchmark dependencies (from source branch)
181+
run: |
182+
python benchmarks/scripts/extract_deps.py /tmp/source_pyproject.toml \
183+
| uv pip install -r - --system
184+
185+
- name: Run integration benchmarks with CodSpeed (walltime)
186+
uses: CodSpeedHQ/action@v4
187+
with:
188+
mode: walltime
189+
run: |
190+
if [ "${{ inputs.benchmark_set }}" = "fast" ]; then
191+
pytest benchmarks/ --codspeed -m "integration and not slow"
192+
else
193+
pytest benchmarks/ --codspeed -m "integration"
194+
fi

‎.github/workflows/benchmarks-integration.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: Benchmarks (Integration)
2+
3+
# Runs integration benchmarks on PRs and main using walltime mode on CodSpeed
4+
# Macro Runners. Walltime accurately measures I/O, network, and system calls.
5+
# See benchmarks/README.md for details.
6+
7+
on:
8+
pull_request:
9+
push:
10+
branches: [main]
11+
12+
permissions:
13+
contents: read
14+
id-token: write
15+
16+
jobs:
17+
benchmark-integration:
18+
runs-on: codspeed-macro
19+
timeout-minutes: 30
20+
21+
steps:
22+
- uses: actions/checkout@v4
23+
with:
24+
fetch-depth: 0
25+
26+
- name: Set up Python
27+
uses: actions/setup-python@v5
28+
with:
29+
python-version: "3.11"
30+
31+
- name: Install hatch
32+
run: pip install hatch
33+
34+
- name: Run integration benchmarks with CodSpeed (walltime)
35+
uses: CodSpeedHQ/action@v4
36+
with:
37+
mode: walltime
38+
run: hatch run benchmark:run --codspeed -m "integration and not slow"
39+
40+
- name: Generate benchmark JSON (fallback)
41+
if: always()
42+
run: |
43+
hatch run benchmark:run \
44+
--benchmark-only \
45+
--benchmark-json=benchmark-results-integration.json \
46+
-m "integration and not slow" || true
47+
48+
- name: Upload benchmark results
49+
uses: actions/upload-artifact@v4
50+
if: always()
51+
with:
52+
name: benchmark-results-integration
53+
path: benchmark-results-integration.json
54+
retention-days: 30

‎.github/workflows/benchmarks-nightly.yml‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
name: Benchmarks (Nightly)
2+
3+
# Runs full benchmark suite nightly or on PRs with 'run-benchmarks' label.
4+
# Component benchmarks use CPU simulation, integration benchmarks use walltime.
5+
# Skips if no commits in 24 hours. See benchmarks/README.md for details.
6+
7+
# ##############################################################################
8+
# DISABLED: No slow benchmarks exist yet. All current benchmarks run in the
9+
# regular CI workflow. To enable: uncomment the schedule and pull_request
10+
# triggers below when slow benchmarks are added.
11+
# ##############################################################################
12+
13+
on:
14+
# schedule:
15+
# - cron: "0 2 * * *" # 2 AM UTC daily
16+
# pull_request:
17+
# types: [labeled]
18+
workflow_dispatch: # Manual trigger only until slow benchmarks exist
19+
20+
permissions:
21+
contents: read
22+
id-token: write
23+
24+
jobs:
25+
check-changes:
26+
runs-on: ubuntu-latest
27+
outputs:
28+
should_run: ${{ steps.check.outputs.should_run }}
29+
steps:
30+
- uses: actions/checkout@v4
31+
with:
32+
fetch-depth: 2
33+
34+
- name: Check if should run
35+
id: check
36+
run: |
37+
# Always run for label triggers and manual dispatch
38+
if [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
39+
echo "should_run=true" >> $GITHUB_OUTPUT
40+
exit 0
41+
fi
42+
43+
# For scheduled runs, skip if HEAD hasn't changed in 24 hours
44+
LAST_COMMIT_TIME=$(git log -1 --format=%ct)
45+
NOW=$(date +%s)
46+
HOURS_AGO=$(( (NOW - LAST_COMMIT_TIME) / 3600 ))
47+
48+
if [ "$HOURS_AGO" -gt 24 ]; then
49+
echo "No commits in the last 24 hours, skipping nightly benchmark"
50+
echo "should_run=false" >> $GITHUB_OUTPUT
51+
else
52+
echo "should_run=true" >> $GITHUB_OUTPUT
53+
fi
54+
55+
# Component benchmarks: CPU-bound, pure Python operations
56+
# Uses CPU simulation for deterministic, hardware-independent measurements
57+
component-benchmarks:
58+
needs: check-changes
59+
if: |
60+
needs.check-changes.outputs.should_run == 'true' &&
61+
(github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks'))
62+
runs-on: ubuntu-latest
63+
timeout-minutes: 60
64+
65+
steps:
66+
- uses: actions/checkout@v4
67+
with:
68+
fetch-depth: 0
69+
70+
- name: Set up Python
71+
uses: actions/setup-python@v5
72+
with:
73+
python-version: "3.11"
74+
75+
- name: Install hatch
76+
run: pip install hatch
77+
78+
- name: Run component benchmarks with CodSpeed
79+
uses: CodSpeedHQ/action@v4
80+
with:
81+
mode: simulation
82+
run: hatch run benchmark:run --codspeed -m "not integration"
83+
84+
- name: Generate benchmark JSON (fallback)
85+
if: always()
86+
run: |
87+
hatch run benchmark:run \
88+
--benchmark-only \
89+
--benchmark-json=benchmark-results-component.json \
90+
-m "not integration" || true
91+
92+
- name: Upload benchmark results
93+
uses: actions/upload-artifact@v4
94+
if: always()
95+
with:
96+
name: benchmark-results-nightly-component
97+
path: benchmark-results-component.json
98+
retention-days: 90
99+
100+
# Integration benchmarks: I/O-bound operations with network and file access
101+
# Uses walltime on Macro Runners for accurate real-world measurements
102+
integration-benchmarks:
103+
needs: check-changes
104+
if: |
105+
needs.check-changes.outputs.should_run == 'true' &&
106+
(github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks'))
107+
runs-on: codspeed-macro
108+
timeout-minutes: 60
109+
110+
steps:
111+
- uses: actions/checkout@v4
112+
with:
113+
fetch-depth: 0
114+
115+
- name: Set up Python
116+
uses: actions/setup-python@v5
117+
with:
118+
python-version: "3.11"
119+
120+
- name: Install hatch
121+
run: pip install hatch
122+
123+
- name: Run integration benchmarks with CodSpeed (walltime)
124+
uses: CodSpeedHQ/action@v4
125+
with:
126+
mode: walltime
127+
run: hatch run benchmark:run --codspeed -m "integration"
128+
129+
- name: Generate benchmark JSON (fallback)
130+
if: always()
131+
run: |
132+
hatch run benchmark:run \
133+
--benchmark-only \
134+
--benchmark-json=benchmark-results-integration.json \
135+
-m "integration" || true
136+
137+
- name: Upload benchmark results
138+
uses: actions/upload-artifact@v4
139+
if: always()
140+
with:
141+
name: benchmark-results-nightly-integration
142+
path: benchmark-results-integration.json
143+
retention-days: 90

0 commit comments

Comments
 (0)