Skip to content

Commit 6fcd542

Browse files
CoreyKaylorCoreyKaylor
committed
Added github actions for benchmarks
1 parent 5805814 commit 6fcd542

2 files changed

Lines changed: 338 additions & 0 deletions

File tree

‎.github/workflows/benchmark-baseline.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: Benchmark Baseline
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- 'src/LightningDB/**'
8+
- 'src/LightningDB.Benchmarks/**'
9+
workflow_dispatch:
10+
11+
jobs:
12+
baseline:
13+
runs-on: ubuntu-latest
14+
timeout-minutes: 30
15+
16+
steps:
17+
- name: Checkout
18+
uses: actions/checkout@v5
19+
20+
- name: Setup .NET
21+
uses: actions/setup-dotnet@v5
22+
with:
23+
dotnet-version: '10.0.x'
24+
25+
- name: Restore dependencies
26+
run: dotnet restore
27+
28+
- name: Build Release
29+
run: dotnet build --configuration Release --no-restore
30+
31+
- name: Run Benchmarks
32+
working-directory: src/LightningDB.Benchmarks
33+
run: |
34+
dotnet run -c Release --no-build -- \
35+
--filter "*" \
36+
--job short \
37+
--exporters json \
38+
--iterationCount 3 \
39+
--warmupCount 1 \
40+
--launchCount 1
41+
42+
- name: Combine benchmark results
43+
run: |
44+
mkdir -p benchmark-cache
45+
# Find all JSON result files and combine them
46+
find src/LightningDB.Benchmarks/BenchmarkDotNet.Artifacts/results \
47+
-name "*-report-full-compressed.json" \
48+
-exec cat {} + > benchmark-cache/baseline.json
49+
50+
- name: Cache baseline results
51+
uses: actions/cache/save@v4
52+
with:
53+
path: benchmark-cache
54+
key: benchmark-baseline-${{ runner.os }}-${{ github.sha }}
55+
56+
- name: Cache baseline results (latest)
57+
uses: actions/cache/save@v4
58+
with:
59+
path: benchmark-cache
60+
key: benchmark-baseline-${{ runner.os }}-latest
61+
62+
- name: Upload baseline artifact
63+
uses: actions/upload-artifact@v4
64+
with:
65+
name: benchmark-baseline-${{ github.sha }}
66+
path: benchmark-cache/
67+
retention-days: 30

‎.github/workflows/benchmark-pr.yml‎

Lines changed: 271 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,271 @@
1+
name: Benchmark PR Check
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
paths:
7+
- 'src/LightningDB/**'
8+
- 'src/LightningDB.Benchmarks/**'
9+
10+
permissions:
11+
contents: read
12+
pull-requests: write
13+
14+
jobs:
15+
benchmark:
16+
runs-on: ubuntu-latest
17+
timeout-minutes: 30
18+
19+
steps:
20+
- name: Checkout PR
21+
uses: actions/checkout@v5
22+
23+
- name: Setup .NET
24+
uses: actions/setup-dotnet@v5
25+
with:
26+
dotnet-version: '10.0.x'
27+
28+
- name: Restore dependencies
29+
run: dotnet restore
30+
31+
- name: Build Release
32+
run: dotnet build --configuration Release --no-restore
33+
34+
- name: Restore baseline cache
35+
id: cache-baseline
36+
uses: actions/cache/restore@v4
37+
with:
38+
path: benchmark-cache
39+
key: benchmark-baseline-${{ runner.os }}-latest
40+
restore-keys: |
41+
benchmark-baseline-${{ runner.os }}-
42+
43+
- name: Check baseline exists
44+
id: check-baseline
45+
run: |
46+
if [ -f "benchmark-cache/baseline.json" ]; then
47+
echo "exists=true" >> $GITHUB_OUTPUT
48+
echo "Baseline found from cache"
49+
else
50+
echo "exists=false" >> $GITHUB_OUTPUT
51+
echo "::warning::No baseline cache found. Benchmarks will run but comparison will be skipped."
52+
fi
53+
54+
- name: Run Benchmarks
55+
working-directory: src/LightningDB.Benchmarks
56+
run: |
57+
dotnet run -c Release --no-build -- \
58+
--filter "*" \
59+
--job short \
60+
--exporters json \
61+
--iterationCount 3 \
62+
--warmupCount 1 \
63+
--launchCount 1
64+
65+
- name: Combine PR benchmark results
66+
run: |
67+
mkdir -p pr-results
68+
find src/LightningDB.Benchmarks/BenchmarkDotNet.Artifacts/results \
69+
-name "*-report-full-compressed.json" \
70+
-exec cat {} + > pr-results/current.json
71+
72+
- name: Compare benchmarks and check for regressions
73+
if: steps.check-baseline.outputs.exists == 'true'
74+
id: compare
75+
run: |
76+
python3 << 'EOF'
77+
import json
78+
import sys
79+
import os
80+
81+
THRESHOLD = 20.0 # 20% regression threshold
82+
83+
def parse_benchmarks(file_path):
84+
"""Parse concatenated BenchmarkDotNet JSON files."""
85+
benchmarks = {}
86+
with open(file_path, 'r') as f:
87+
content = f.read()
88+
89+
# Handle concatenated JSON objects (one per benchmark class)
90+
decoder = json.JSONDecoder()
91+
pos = 0
92+
while pos < len(content):
93+
# Skip whitespace
94+
while pos < len(content) and content[pos] in ' \t\n\r':
95+
pos += 1
96+
if pos >= len(content):
97+
break
98+
try:
99+
obj, end = decoder.raw_decode(content, pos)
100+
pos = end
101+
if 'Benchmarks' in obj:
102+
for b in obj['Benchmarks']:
103+
name = b.get('FullName', b.get('Method', 'unknown'))
104+
if 'Statistics' in b and 'Mean' in b['Statistics']:
105+
benchmarks[name] = b['Statistics']['Mean']
106+
except json.JSONDecodeError:
107+
pos += 1
108+
109+
return benchmarks
110+
111+
baseline = parse_benchmarks('benchmark-cache/baseline.json')
112+
current = parse_benchmarks('pr-results/current.json')
113+
114+
print(f"Baseline benchmarks: {len(baseline)}")
115+
print(f"Current benchmarks: {len(current)}")
116+
117+
regressions = []
118+
improvements = []
119+
results = []
120+
121+
for name, current_mean in current.items():
122+
if name in baseline:
123+
baseline_mean = baseline[name]
124+
if baseline_mean > 0:
125+
change_pct = ((current_mean - baseline_mean) / baseline_mean) * 100
126+
127+
results.append({
128+
'name': name,
129+
'baseline': baseline_mean,
130+
'current': current_mean,
131+
'change': change_pct
132+
})
133+
134+
if change_pct > THRESHOLD:
135+
regressions.append({
136+
'name': name,
137+
'baseline': baseline_mean,
138+
'current': current_mean,
139+
'change': change_pct
140+
})
141+
elif change_pct < -THRESHOLD:
142+
improvements.append({
143+
'name': name,
144+
'change': change_pct
145+
})
146+
147+
# Write results for PR comment
148+
with open('pr-results/comparison.json', 'w') as f:
149+
json.dump(results, f, indent=2)
150+
151+
# Generate summary
152+
print(f"\nCompared {len(results)} benchmarks")
153+
print(f"Regressions (>{THRESHOLD}%): {len(regressions)}")
154+
print(f"Improvements (<-{THRESHOLD}%): {len(improvements)}")
155+
156+
if regressions:
157+
print("\n::error::Performance regressions detected!")
158+
for r in sorted(regressions, key=lambda x: -x['change']):
159+
print(f" - {r['name']}: +{r['change']:.1f}% slower ({r['baseline']:.2f}ns -> {r['current']:.2f}ns)")
160+
161+
with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
162+
f.write("has_regressions=true\n")
163+
f.write(f"regression_count={len(regressions)}\n")
164+
sys.exit(1)
165+
else:
166+
print("\nNo significant performance regressions detected.")
167+
with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
168+
f.write("has_regressions=false\n")
169+
EOF
170+
171+
- name: Comment PR with benchmark results
172+
if: always() && steps.check-baseline.outputs.exists == 'true' && hashFiles('pr-results/comparison.json') != ''
173+
uses: actions/github-script@v7
174+
with:
175+
script: |
176+
const fs = require('fs');
177+
178+
let body = '## Benchmark Results\n\n';
179+
180+
try {
181+
const results = JSON.parse(fs.readFileSync('pr-results/comparison.json', 'utf8'));
182+
183+
// Sort by change percentage (worst regressions first)
184+
results.sort((a, b) => b.change - a.change);
185+
186+
const regressions = results.filter(r => r.change > 20);
187+
const warnings = results.filter(r => r.change > 10 && r.change <= 20);
188+
const improvements = results.filter(r => r.change < -10);
189+
190+
if (regressions.length > 0) {
191+
body += ':x: **Performance regressions detected (>20%)**\n\n';
192+
body += '| Benchmark | Baseline | Current | Change |\n';
193+
body += '|-----------|----------|---------|--------|\n';
194+
for (const r of regressions.slice(0, 10)) {
195+
const shortName = r.name.split('.').slice(-2).join('.');
196+
body += `| ${shortName} | ${r.baseline.toFixed(2)}ns | ${r.current.toFixed(2)}ns | :x: +${r.change.toFixed(1)}% |\n`;
197+
}
198+
if (regressions.length > 10) {
199+
body += `\n*...and ${regressions.length - 10} more regressions*\n`;
200+
}
201+
} else {
202+
body += ':white_check_mark: **No significant performance regressions detected**\n\n';
203+
}
204+
205+
if (warnings.length > 0) {
206+
body += `\n### :warning: Minor regressions (10-20%)\n`;
207+
body += `${warnings.length} benchmarks showed minor slowdown\n`;
208+
}
209+
210+
if (improvements.length > 0) {
211+
body += `\n### :rocket: Improvements\n`;
212+
body += `${improvements.length} benchmarks showed improvement (>10% faster)\n`;
213+
}
214+
215+
body += `\n<details><summary>All results (${results.length} benchmarks)</summary>\n\n`;
216+
body += '| Benchmark | Change |\n|-----------|--------|\n';
217+
for (const r of results) {
218+
const shortName = r.name.split('.').slice(-2).join('.');
219+
const emoji = r.change > 20 ? ':x:' : r.change > 10 ? ':warning:' : r.change < -10 ? ':rocket:' : ':white_check_mark:';
220+
const sign = r.change > 0 ? '+' : '';
221+
body += `| ${shortName} | ${emoji} ${sign}${r.change.toFixed(1)}% |\n`;
222+
}
223+
body += '</details>\n';
224+
225+
} catch (e) {
226+
body += ':warning: Could not parse benchmark comparison results.\n';
227+
body += `Error: ${e.message}\n`;
228+
}
229+
230+
// Find existing comment to update
231+
const { data: comments } = await github.rest.issues.listComments({
232+
issue_number: context.issue.number,
233+
owner: context.repo.owner,
234+
repo: context.repo.repo
235+
});
236+
237+
const botComment = comments.find(c =>
238+
c.user.type === 'Bot' && c.body.includes('## Benchmark Results')
239+
);
240+
241+
if (botComment) {
242+
await github.rest.issues.updateComment({
243+
comment_id: botComment.id,
244+
owner: context.repo.owner,
245+
repo: context.repo.repo,
246+
body: body
247+
});
248+
} else {
249+
await github.rest.issues.createComment({
250+
issue_number: context.issue.number,
251+
owner: context.repo.owner,
252+
repo: context.repo.repo,
253+
body: body
254+
});
255+
}
256+
257+
- name: Upload benchmark results
258+
if: always()
259+
uses: actions/upload-artifact@v4
260+
with:
261+
name: benchmark-results-pr-${{ github.event.pull_request.number }}
262+
path: |
263+
pr-results/
264+
src/LightningDB.Benchmarks/BenchmarkDotNet.Artifacts/results/
265+
retention-days: 14
266+
267+
- name: Fail if regressions detected
268+
if: steps.compare.outputs.has_regressions == 'true'
269+
run: |
270+
echo "::error::PR blocked due to performance regressions exceeding 20% threshold"
271+
exit 1

0 commit comments

Comments
 (0)