Establish performance baselines and regression detection

.github/workflows/benchmark.yaml

@@ -0,0 +1,163 @@
+# Copyright The Conforma Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+---
+name: Stress Benchmark
+
+"on":
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+
+  stress:
+    name: Stress Benchmark
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    continue-on-error: false
+    env:
+      # Tuned for 4 vCPU / 16 GB CI runners to complete within 6-8 minutes.
+      # Code defaults are 10 components / 35 workers.
+      EC_STRESS_COMPONENTS: "10"
+      EC_STRESS_WORKERS: "10"
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@58077d3c7e43986b6b15fba718e8ea69e387dfcc # v2.15.1
+        with:
+          egress-policy: audit
+          disable-telemetry: true
+
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Restore Cache
+        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          key: main
+          path: '**'
+
+      - name: Setup Go environment
+        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Setup ORAS
+        uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1
+
+      - name: Prepare benchmark data
+        run: |
+          cd benchmark/stress
+          ./prepare_data.sh
+
+      - name: Build stress benchmark
+        run: go build -o benchmark/stress/stress ./benchmark/stress
+
+      - name: Run stress benchmark
+        id: bench
+        run: |
+          set -o pipefail
+          cd benchmark/stress
+          ./stress 2>benchmark-stderr.txt | tee benchmark-output.txt
+
+      - name: Compare against baseline
+        id: compare
+        run: |
+          cd benchmark/stress
+          if [[ -f baseline.json ]]; then
+            ./compare.sh benchmark-output.txt
+          else
+            echo "No baseline found, skipping comparison."
+          fi
+
+      - name: Write job summary
+        if: always()
+        run: |
+          if [[ "${{ steps.bench.outcome }}" == "failure" && -s benchmark/stress/benchmark-stderr.txt ]]; then
+            {
+              echo "### Stderr"
+              echo '```'
+              tail -50 benchmark/stress/benchmark-stderr.txt
+              echo '```'
+            } >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+          if [[ ! -f benchmark/stress/benchmark-output.txt ]]; then
+            echo "## Stress Benchmark" >> "$GITHUB_STEP_SUMMARY"
+            echo "Benchmark did not produce output." >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          line=$(grep '^BenchmarkStress' benchmark/stress/benchmark-output.txt || true)
+          if [[ -z "$line" ]]; then
+            echo "## Stress Benchmark" >> "$GITHUB_STEP_SUMMARY"
+            echo "No benchmark results found in output." >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          ns_op=$(echo "$line" | grep -oP '[\d.]+ ns/op' | awk '{print $1}')
+          peak_rss=$(echo "$line" | grep -oP '[\d.]+ peak-RSS-bytes' | awk '{print $1}')
+          alloc=$(echo "$line" | grep -oP '[\d.]+ allocated-bytes/op' | awk '{print $1}')
+          heap=$(echo "$line" | grep -oP '[\d.]+ heap-bytes-from-system' | awk '{print $1}')
+
+          secs=$(awk -v val="${ns_op:-0}" 'BEGIN {printf "%.1f", val / 1000000000}')
+          rss_mb=$(awk -v val="${peak_rss:-0}" 'BEGIN {printf "%.0f", val / 1048576}')
+          alloc_mb=$(awk -v val="${alloc:-0}" 'BEGIN {printf "%.0f", val / 1048576}')
+          heap_mb=$(awk -v val="${heap:-0}" 'BEGIN {printf "%.0f", val / 1048576}')
+
+          has_baseline=false
+          if [[ -f benchmark/stress/baseline.json ]]; then
+            has_baseline=true
+            bl_rss=$(python3 -c "import json; print(json.load(open('benchmark/stress/baseline.json'))['peak_rss_bytes'])")
+            bl_ns=$(python3 -c "import json; print(json.load(open('benchmark/stress/baseline.json'))['execution_time_ns'])")
+            bl_rss_mb=$(awk -v val="$bl_rss" 'BEGIN {printf "%.0f", val / 1048576}')
+            bl_secs=$(awk -v val="$bl_ns" 'BEGIN {printf "%.1f", val / 1000000000}')
+            rss_change=$(awk -v cur="$peak_rss" -v base="$bl_rss" 'BEGIN {printf "%+.1f", ((cur - base) / base) * 100}')
+            time_change=$(awk -v cur="$ns_op" -v base="$bl_ns" 'BEGIN {printf "%+.1f", ((cur - base) / base) * 100}')
+          fi
+
+          {
+            echo "## Stress Benchmark"
+            echo ""
+            if [[ "$has_baseline" == "true" ]]; then
+              echo "| Metric | Current | Baseline | Change | Description |"
+              echo "|--------|---------|----------|--------|-------------|"
+              echo "| Components | ${EC_STRESS_COMPONENTS} | | | Snapshot components validated |"
+              echo "| Workers | ${EC_STRESS_WORKERS} | | | Parallel validation workers |"
+              echo "| Execution time | ${secs}s | ${bl_secs}s | ${time_change}% | Wall-clock time per iteration |"
+              echo "| Peak RSS | ${rss_mb} MB | ${bl_rss_mb} MB | ${rss_change}% | Max physical memory used |"
+              echo "| Allocated memory | ${alloc_mb} MB | | | Total Go heap allocations |"
+              echo "| Heap from system | ${heap_mb} MB | | | Heap memory requested from OS |"
+            else
+              echo "| Metric | Value | Description |"
+              echo "|--------|-------|-------------|"
+              echo "| Components | ${EC_STRESS_COMPONENTS} | Snapshot components validated |"
+              echo "| Workers | ${EC_STRESS_WORKERS} | Parallel validation workers |"
+              echo "| Execution time | ${secs}s | Wall-clock time per iteration |"
+              echo "| Peak RSS | ${rss_mb} MB | Max physical memory used |"
+              echo "| Allocated memory | ${alloc_mb} MB | Total Go heap allocations |"
+              echo "| Heap from system | ${heap_mb} MB | Heap memory requested from OS |"
+            fi
+            if [[ "${{ steps.compare.outcome }}" == "failure" ]]; then
+              echo ""
+              echo "> **⚠️ Performance regression detected.** Update the baseline with \`make benchmark_baseline\` if this is expected."
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"

Makefile

@@ -197,6 +197,21 @@ benchmark_data: benchmark/simple/data.tar.gz ## Prepare data for benchmark
 .PHONY: benchmark
 benchmark: benchmark_simple ## Run benchmarks
 
+.PHONY: benchmark_baseline
+benchmark_baseline: benchmark/stress/data.tar.gz ## Generate stress benchmark baseline
+	@cd benchmark/stress && \
+	go run . 2>/dev/null | tee benchmark-output.txt && \
+	line=$$(grep '^BenchmarkStress' benchmark-output.txt) && \
+	ns_op=$$(echo "$$line" | grep -oP '[\d.]+ ns/op' | awk '{print $$1}') && \
+	peak_rss=$$(echo "$$line" | grep -oP '[\d.]+ peak-RSS-bytes' | awk '{print $$1}') && \
+	printf '{\n  "peak_rss_bytes": %s,\n  "execution_time_ns": %s,\n  "components": %s,\n  "workers": %s,\n  "commit": "%s",\n  "date": "%s",\n  "go_version": "%s"\n}\n' \
+		"$$peak_rss" "$$ns_op" \
+		"$${EC_STRESS_COMPONENTS:-10}" "$${EC_STRESS_WORKERS:-35}" \
+		"$$(git rev-parse --short HEAD)" "$$(date -u +%Y-%m-%d)" "$$(go env GOVERSION | sed 's/^go//')" \
+		> baseline.json && \
+	rm -f benchmark-output.txt && \
+	echo "Baseline written to benchmark/stress/baseline.json"
+
 .PHONY: tools-ci
 tools-ci: ## Ensure all tools build cleanly
 	@echo "• tkn:" && \

benchmark/stress/baseline.json

@@ -0,0 +1,9 @@
+{
+  "peak_rss_bytes": 2254512128,
+  "execution_time_ns": 2740282760,
+  "components": 10,
+  "workers": 10,
+  "commit": "45c5b81c",
+  "date": "2026-07-27",
+  "go_version": "1.26.3"
+}

benchmark/stress/compare.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+# Copyright The Conforma Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Compares current benchmark results against a stored baseline and exits
+# non-zero if any metric regresses beyond the configured threshold.
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BASELINE="${SCRIPT_DIR}/baseline.json"
+THRESHOLDS="${SCRIPT_DIR}/thresholds.json"
+BENCHMARK_OUTPUT="${1:-${SCRIPT_DIR}/benchmark-output.txt}"
+
+if [[ ! -f "$BASELINE" ]]; then
+    echo "No baseline found, skipping comparison."
+    exit 0
+fi
+
+if [[ ! -f "$THRESHOLDS" ]]; then
+    echo "No thresholds file found, skipping comparison."
+    exit 0
+fi
+
+if [[ ! -f "$BENCHMARK_OUTPUT" ]]; then
+    echo "No benchmark output found at ${BENCHMARK_OUTPUT}"
+    exit 1
+fi
+
+line=$(grep '^BenchmarkStress' "$BENCHMARK_OUTPUT" || true)
+if [[ -z "$line" ]]; then
+    echo "No BenchmarkStress results found in output."
+    exit 1
+fi
+
+current_ns=$(echo "$line" | grep -oP '[\d.]+ ns/op' | awk '{print $1}')
+current_rss=$(echo "$line" | grep -oP '[\d.]+ peak-RSS-bytes' | awk '{print $1}')
+
+baseline_ns=$(python3 -c "import json; print(json.load(open('${BASELINE}'))['execution_time_ns'])")
+baseline_rss=$(python3 -c "import json; print(json.load(open('${BASELINE}'))['peak_rss_bytes'])")
+threshold_rss=$(python3 -c "import json; print(json.load(open('${THRESHOLDS}'))['peak_rss_percent'])")
+threshold_time=$(python3 -c "import json; print(json.load(open('${THRESHOLDS}'))['execution_time_percent'])")
+
+rss_change=$(awk -v cur="$current_rss" -v base="$baseline_rss" 'BEGIN {printf "%.1f", ((cur - base) / base) * 100}')
+time_change=$(awk -v cur="$current_ns" -v base="$baseline_ns" 'BEGIN {printf "%.1f", ((cur - base) / base) * 100}')
+
+baseline_rss_mb=$(awk -v val="$baseline_rss" 'BEGIN {printf "%.0f", val / 1048576}')
+current_rss_mb=$(awk -v val="$current_rss" 'BEGIN {printf "%.0f", val / 1048576}')
+baseline_secs=$(awk -v val="$baseline_ns" 'BEGIN {printf "%.1f", val / 1000000000}')
+current_secs=$(awk -v val="$current_ns" 'BEGIN {printf "%.1f", val / 1000000000}')
+
+echo ""
+echo "=== Benchmark Comparison ==="
+echo ""
+printf "%-20s %10s %10s %10s %10s\n" "Metric" "Baseline" "Current" "Change" "Threshold"
+printf "%-20s %10s %10s %9s%% %9s%%\n" "Peak RSS" "${baseline_rss_mb} MB" "${current_rss_mb} MB" "$rss_change" "$threshold_rss"
+printf "%-20s %10s %10s %9s%% %9s%%\n" "Execution time" "${baseline_secs}s" "${current_secs}s" "$time_change" "$threshold_time"
+echo ""
+
+failed=0
+
+rss_exceeded=$(awk -v change="$rss_change" -v thresh="$threshold_rss" 'BEGIN {print (change > thresh) ? 1 : 0}')
+time_exceeded=$(awk -v change="$time_change" -v thresh="$threshold_time" 'BEGIN {print (change > thresh) ? 1 : 0}')
+
+if [[ "$rss_exceeded" == "1" ]]; then
+    echo "FAIL: Peak RSS regressed by ${rss_change}% (threshold: ${threshold_rss}%)"
+    failed=1
+fi
+
+if [[ "$time_exceeded" == "1" ]]; then
+    echo "FAIL: Execution time regressed by ${time_change}% (threshold: ${threshold_time}%)"
+    failed=1
+fi
+
+if [[ "$failed" == "0" ]]; then
+    echo "PASS: No regressions detected."
+fi
+
+exit "$failed"

benchmark/stress/thresholds.json

@@ -0,0 +1,4 @@
+{
+  "peak_rss_percent": 15,
+  "execution_time_percent": 20
+}