diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 000000000..464fffb5a --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,163 @@ +# Copyright The Conforma Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +--- +name: Stress Benchmark + +"on": + pull_request: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + + stress: + name: Stress Benchmark + runs-on: ubuntu-latest + timeout-minutes: 15 + continue-on-error: false + env: + # Tuned for 4 vCPU / 16 GB CI runners to complete within 6-8 minutes. + # Code defaults are 10 components / 35 workers. + EC_STRESS_COMPONENTS: "10" + EC_STRESS_WORKERS: "10" + steps: + - name: Harden Runner + uses: step-security/harden-runner@58077d3c7e43986b6b15fba718e8ea69e387dfcc # v2.15.1 + with: + egress-policy: audit + disable-telemetry: true + + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Restore Cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + key: main + path: '**' + + - name: Setup Go environment + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + cache: false + + - name: Setup ORAS + uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 + + - name: Prepare benchmark data + run: | + cd benchmark/stress + ./prepare_data.sh + + - name: Build stress benchmark + run: go build -o benchmark/stress/stress ./benchmark/stress + + - name: Run stress benchmark + id: bench + run: | + set -o pipefail + cd benchmark/stress + ./stress 2>benchmark-stderr.txt | tee benchmark-output.txt + + - name: Compare against baseline + id: compare + run: | + cd benchmark/stress + if [[ -f baseline.json ]]; then + ./compare.sh benchmark-output.txt + else + echo "No baseline found, skipping comparison." + fi + + - name: Write job summary + if: always() + run: | + if [[ "${{ steps.bench.outcome }}" == "failure" && -s benchmark/stress/benchmark-stderr.txt ]]; then + { + echo "### Stderr" + echo '```' + tail -50 benchmark/stress/benchmark-stderr.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + fi + + if [[ ! -f benchmark/stress/benchmark-output.txt ]]; then + echo "## Stress Benchmark" >> "$GITHUB_STEP_SUMMARY" + echo "Benchmark did not produce output." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + line=$(grep '^BenchmarkStress' benchmark/stress/benchmark-output.txt || true) + if [[ -z "$line" ]]; then + echo "## Stress Benchmark" >> "$GITHUB_STEP_SUMMARY" + echo "No benchmark results found in output." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + ns_op=$(echo "$line" | grep -oP '[\d.]+ ns/op' | awk '{print $1}') + peak_rss=$(echo "$line" | grep -oP '[\d.]+ peak-RSS-bytes' | awk '{print $1}') + alloc=$(echo "$line" | grep -oP '[\d.]+ allocated-bytes/op' | awk '{print $1}') + heap=$(echo "$line" | grep -oP '[\d.]+ heap-bytes-from-system' | awk '{print $1}') + + secs=$(awk -v val="${ns_op:-0}" 'BEGIN {printf "%.1f", val / 1000000000}') + rss_mb=$(awk -v val="${peak_rss:-0}" 'BEGIN {printf "%.0f", val / 1048576}') + alloc_mb=$(awk -v val="${alloc:-0}" 'BEGIN {printf "%.0f", val / 1048576}') + heap_mb=$(awk -v val="${heap:-0}" 'BEGIN {printf "%.0f", val / 1048576}') + + has_baseline=false + if [[ -f benchmark/stress/baseline.json ]]; then + has_baseline=true + bl_rss=$(python3 -c "import json; print(json.load(open('benchmark/stress/baseline.json'))['peak_rss_bytes'])") + bl_ns=$(python3 -c "import json; print(json.load(open('benchmark/stress/baseline.json'))['execution_time_ns'])") + bl_rss_mb=$(awk -v val="$bl_rss" 'BEGIN {printf "%.0f", val / 1048576}') + bl_secs=$(awk -v val="$bl_ns" 'BEGIN {printf "%.1f", val / 1000000000}') + rss_change=$(awk -v cur="$peak_rss" -v base="$bl_rss" 'BEGIN {printf "%+.1f", ((cur - base) / base) * 100}') + time_change=$(awk -v cur="$ns_op" -v base="$bl_ns" 'BEGIN {printf "%+.1f", ((cur - base) / base) * 100}') + fi + + { + echo "## Stress Benchmark" + echo "" + if [[ "$has_baseline" == "true" ]]; then + echo "| Metric | Current | Baseline | Change | Description |" + echo "|--------|---------|----------|--------|-------------|" + echo "| Components | ${EC_STRESS_COMPONENTS} | | | Snapshot components validated |" + echo "| Workers | ${EC_STRESS_WORKERS} | | | Parallel validation workers |" + echo "| Execution time | ${secs}s | ${bl_secs}s | ${time_change}% | Wall-clock time per iteration |" + echo "| Peak RSS | ${rss_mb} MB | ${bl_rss_mb} MB | ${rss_change}% | Max physical memory used |" + echo "| Allocated memory | ${alloc_mb} MB | | | Total Go heap allocations |" + echo "| Heap from system | ${heap_mb} MB | | | Heap memory requested from OS |" + else + echo "| Metric | Value | Description |" + echo "|--------|-------|-------------|" + echo "| Components | ${EC_STRESS_COMPONENTS} | Snapshot components validated |" + echo "| Workers | ${EC_STRESS_WORKERS} | Parallel validation workers |" + echo "| Execution time | ${secs}s | Wall-clock time per iteration |" + echo "| Peak RSS | ${rss_mb} MB | Max physical memory used |" + echo "| Allocated memory | ${alloc_mb} MB | Total Go heap allocations |" + echo "| Heap from system | ${heap_mb} MB | Heap memory requested from OS |" + fi + if [[ "${{ steps.compare.outcome }}" == "failure" ]]; then + echo "" + echo "> **⚠️ Performance regression detected.** Update the baseline with \`make benchmark_baseline\` if this is expected." + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/Makefile b/Makefile index 9f094e69e..42f0adea1 100644 --- a/Makefile +++ b/Makefile @@ -197,6 +197,21 @@ benchmark_data: benchmark/simple/data.tar.gz ## Prepare data for benchmark .PHONY: benchmark benchmark: benchmark_simple ## Run benchmarks +.PHONY: benchmark_baseline +benchmark_baseline: benchmark/stress/data.tar.gz ## Generate stress benchmark baseline + @cd benchmark/stress && \ + go run . 2>/dev/null | tee benchmark-output.txt && \ + line=$$(grep '^BenchmarkStress' benchmark-output.txt) && \ + ns_op=$$(echo "$$line" | grep -oP '[\d.]+ ns/op' | awk '{print $$1}') && \ + peak_rss=$$(echo "$$line" | grep -oP '[\d.]+ peak-RSS-bytes' | awk '{print $$1}') && \ + printf '{\n "peak_rss_bytes": %s,\n "execution_time_ns": %s,\n "components": %s,\n "workers": %s,\n "commit": "%s",\n "date": "%s",\n "go_version": "%s"\n}\n' \ + "$$peak_rss" "$$ns_op" \ + "$${EC_STRESS_COMPONENTS:-10}" "$${EC_STRESS_WORKERS:-35}" \ + "$$(git rev-parse --short HEAD)" "$$(date -u +%Y-%m-%d)" "$$(go env GOVERSION | sed 's/^go//')" \ + > baseline.json && \ + rm -f benchmark-output.txt && \ + echo "Baseline written to benchmark/stress/baseline.json" + .PHONY: tools-ci tools-ci: ## Ensure all tools build cleanly @echo "• tkn:" && \ diff --git a/benchmark/stress/baseline.json b/benchmark/stress/baseline.json new file mode 100644 index 000000000..60c0ff87e --- /dev/null +++ b/benchmark/stress/baseline.json @@ -0,0 +1,9 @@ +{ + "peak_rss_bytes": 2254512128, + "execution_time_ns": 2740282760, + "components": 10, + "workers": 10, + "commit": "45c5b81c", + "date": "2026-07-27", + "go_version": "1.26.3" +} diff --git a/benchmark/stress/compare.sh b/benchmark/stress/compare.sh new file mode 100755 index 000000000..7318720dc --- /dev/null +++ b/benchmark/stress/compare.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Copyright The Conforma Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# Compares current benchmark results against a stored baseline and exits +# non-zero if any metric regresses beyond the configured threshold. +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BASELINE="${SCRIPT_DIR}/baseline.json" +THRESHOLDS="${SCRIPT_DIR}/thresholds.json" +BENCHMARK_OUTPUT="${1:-${SCRIPT_DIR}/benchmark-output.txt}" + +if [[ ! -f "$BASELINE" ]]; then + echo "No baseline found, skipping comparison." + exit 0 +fi + +if [[ ! -f "$THRESHOLDS" ]]; then + echo "No thresholds file found, skipping comparison." + exit 0 +fi + +if [[ ! -f "$BENCHMARK_OUTPUT" ]]; then + echo "No benchmark output found at ${BENCHMARK_OUTPUT}" + exit 1 +fi + +line=$(grep '^BenchmarkStress' "$BENCHMARK_OUTPUT" || true) +if [[ -z "$line" ]]; then + echo "No BenchmarkStress results found in output." + exit 1 +fi + +current_ns=$(echo "$line" | grep -oP '[\d.]+ ns/op' | awk '{print $1}') +current_rss=$(echo "$line" | grep -oP '[\d.]+ peak-RSS-bytes' | awk '{print $1}') + +baseline_ns=$(python3 -c "import json; print(json.load(open('${BASELINE}'))['execution_time_ns'])") +baseline_rss=$(python3 -c "import json; print(json.load(open('${BASELINE}'))['peak_rss_bytes'])") +threshold_rss=$(python3 -c "import json; print(json.load(open('${THRESHOLDS}'))['peak_rss_percent'])") +threshold_time=$(python3 -c "import json; print(json.load(open('${THRESHOLDS}'))['execution_time_percent'])") + +rss_change=$(awk -v cur="$current_rss" -v base="$baseline_rss" 'BEGIN {printf "%.1f", ((cur - base) / base) * 100}') +time_change=$(awk -v cur="$current_ns" -v base="$baseline_ns" 'BEGIN {printf "%.1f", ((cur - base) / base) * 100}') + +baseline_rss_mb=$(awk -v val="$baseline_rss" 'BEGIN {printf "%.0f", val / 1048576}') +current_rss_mb=$(awk -v val="$current_rss" 'BEGIN {printf "%.0f", val / 1048576}') +baseline_secs=$(awk -v val="$baseline_ns" 'BEGIN {printf "%.1f", val / 1000000000}') +current_secs=$(awk -v val="$current_ns" 'BEGIN {printf "%.1f", val / 1000000000}') + +echo "" +echo "=== Benchmark Comparison ===" +echo "" +printf "%-20s %10s %10s %10s %10s\n" "Metric" "Baseline" "Current" "Change" "Threshold" +printf "%-20s %10s %10s %9s%% %9s%%\n" "Peak RSS" "${baseline_rss_mb} MB" "${current_rss_mb} MB" "$rss_change" "$threshold_rss" +printf "%-20s %10s %10s %9s%% %9s%%\n" "Execution time" "${baseline_secs}s" "${current_secs}s" "$time_change" "$threshold_time" +echo "" + +failed=0 + +rss_exceeded=$(awk -v change="$rss_change" -v thresh="$threshold_rss" 'BEGIN {print (change > thresh) ? 1 : 0}') +time_exceeded=$(awk -v change="$time_change" -v thresh="$threshold_time" 'BEGIN {print (change > thresh) ? 1 : 0}') + +if [[ "$rss_exceeded" == "1" ]]; then + echo "FAIL: Peak RSS regressed by ${rss_change}% (threshold: ${threshold_rss}%)" + failed=1 +fi + +if [[ "$time_exceeded" == "1" ]]; then + echo "FAIL: Execution time regressed by ${time_change}% (threshold: ${threshold_time}%)" + failed=1 +fi + +if [[ "$failed" == "0" ]]; then + echo "PASS: No regressions detected." +fi + +exit "$failed" diff --git a/benchmark/stress/thresholds.json b/benchmark/stress/thresholds.json new file mode 100644 index 000000000..9a29e7315 --- /dev/null +++ b/benchmark/stress/thresholds.json @@ -0,0 +1,4 @@ +{ + "peak_rss_percent": 15, + "execution_time_percent": 20 +}