From 4d57c0f0ebdd9e3ea3cdab76fef51a94a8239bc1 Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Thu, 20 Aug 2026 10:12:40 +0200 Subject: [PATCH] vm-tests: add pgbench PostgreSQL kernel A/B regression test Add a pgbench-kernel-regression VM test that installs a base kernel, runs a PostgreSQL 16 pgbench read-only and read-write suite, reboots into a second (tip) kernel, re-runs the suite, and emits benchmark-base-*.csv and benchmark-tip-*.csv for the regression analyzer to compare (read-only/read-write TPS and average latency). Signed-off-by: Norbert Manthey --- .gitignore | 1 + vm-tests/README.md | 1 + vm-tests/pgbench-kernel-regression/README.md | 58 +++++ .../pgbench-kernel-regression/common_lib.sh | 213 ++++++++++++++++++ .../dependencies.txt | 5 + .../external_requirements.json | 4 + .../kernel_helpers.sh | 1 + .../package_helpers.sh | 1 + .../run-01-setup-kernel-A.sh | 32 +++ .../run-02-run-pgbench-setup-kernel-B.sh | 41 ++++ .../run-03-run-second-pgbench.sh | 35 +++ 11 files changed, 392 insertions(+) create mode 100644 vm-tests/pgbench-kernel-regression/README.md create mode 100644 vm-tests/pgbench-kernel-regression/common_lib.sh create mode 100644 vm-tests/pgbench-kernel-regression/dependencies.txt create mode 100644 vm-tests/pgbench-kernel-regression/external_requirements.json create mode 120000 vm-tests/pgbench-kernel-regression/kernel_helpers.sh create mode 120000 vm-tests/pgbench-kernel-regression/package_helpers.sh create mode 100755 vm-tests/pgbench-kernel-regression/run-01-setup-kernel-A.sh create mode 100755 vm-tests/pgbench-kernel-regression/run-02-run-pgbench-setup-kernel-B.sh create mode 100755 vm-tests/pgbench-kernel-regression/run-03-run-second-pgbench.sh diff --git a/.gitignore b/.gitignore index 7ea422a..de34e0c 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ examples/aws/credentials.json # Project-specific config (generated by setup configure) run-config.json demo-config.json +pgbench-config.json # Test kernel RPMs (large binary files) setup/test-kernel-rpms/ diff --git a/vm-tests/README.md b/vm-tests/README.md index 1180965..725936e 100644 --- a/vm-tests/README.md +++ b/vm-tests/README.md @@ -12,6 +12,7 @@ See the main [README](../README.md) for writing new tests, configuration, and th | `example-kernel-reboot-test` | 3 | yes | Installs two kernels with reboot between each | | `simple-unixbench` | 1 | no | Runs UnixBench on the default kernel | | `unixbench-kernel-regression` | 3 | yes | Installs two kernels, runs UnixBench on each, produces benchmark CSVs | +| `pgbench-kernel-regression` | 3 | yes | Installs two kernels, runs PostgreSQL pgbench (read-only + read-write) on each, produces benchmark CSVs | | `simple-source-reboot` | 2 | yes | Installs kernel from source RPM, reboots, verifies | ## How Multi-Stage Tests Work diff --git a/vm-tests/pgbench-kernel-regression/README.md b/vm-tests/pgbench-kernel-regression/README.md new file mode 100644 index 0000000..3adda28 --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/README.md @@ -0,0 +1,58 @@ +# PostgreSQL pgbench Kernel Regression Test + +Kernel A/B performance regression test using PostgreSQL's `pgbench`. It installs +two kernels in turn on a **single VM** and runs the same read-only and +read-write `pgbench` workload against each, so the pipeline's benchmark analyzer +can flag database-performance regressions between kernel versions. It follows +the same three-stage pattern as `unixbench-kernel-regression`. + +- Dependencies are installed from `dependencies.txt` at run time (not from test + metadata). +- Results are emitted as `benchmark-*.csv` in the schema the pipeline already + supports + +## Test Flow + +1. **run-01-setup-kernel-A.sh** — install PostgreSQL packages, record the running + kernel, install the first (lowest-version) kernel RPM from the shared + `kernel-rpms` area, then reboot. +2. **run-02-run-pgbench-setup-kernel-B.sh** — confirm the kernel changed, run + `pgbench` (read-only + read-write) on the base kernel, then install the second + (highest-version) kernel and reboot. +3. **run-03-run-second-pgbench.sh** — confirm the kernel changed, run `pgbench` + on the tip kernel. + +Single VM, CPU-pinned: PostgreSQL is pinned to the first half of the cores and +the `pgbench` client to the second half, to reduce client/server interference. + +## Output + +- `benchmark-base-.csv` — metrics for the first (base) kernel. +- `benchmark-tip-.csv` — metrics for the second (tip) kernel. + +CSV columns: `metric,unit,value,more_is_better,kernel_version,instance_id,instance_type,arch` + +Metrics captured (per kernel): +- `postgresql.readonly.tps` / `postgresql.readwrite.tps`: transactions/sec (more is better). +- `postgresql.readonly.latency_avg` / `postgresql.readwrite.latency_avg`: average latency in ms (less is better). + +The pipeline's benchmark analyzer compares the base and tip CSVs and reports +regressions. + +## Requirements + +- x86_64 or aarch64 instance with at least 4 vCPUs (CPU pinning splits + server/client across the two halves); `c8i.4xlarge` recommended, `us-west-2` + preferred to raise the chance of landing on the same hardware for both kernels. +- Two kernel RPMs uploaded to the shared kernel-rpms area + (`external_requirements.json` sets `kernel-rpms/binary: true`). The lowest + version becomes the base, the highest becomes the tip. +- System packages from `dependencies.txt` (`postgresql16-server`, + `postgresql16-contrib`, `postgresql16`) are installed in run-01. + +## Configuration (environment overrides) + +| Variable | Default | Purpose | +|---|---|---| +| `PGBENCH_DURATION` | `240` | Duration in seconds of each pgbench run. | +| `PGBENCH_SCALING_FACTOR` | `100` | Database size multiplier passed to `pgbench -i -s`. | diff --git a/vm-tests/pgbench-kernel-regression/common_lib.sh b/vm-tests/pgbench-kernel-regression/common_lib.sh new file mode 100644 index 0000000..4d27ac4 --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/common_lib.sh @@ -0,0 +1,213 @@ +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Common library for the pgbench PostgreSQL kernel A/B benchmark. +# +# The kernel-management half (install first/last kernel from the shared +# kernel-rpms area, reboot between stages, assert the running kernel changed) +# is reused from the unixbench-kernel-regression test so behaviour stays +# consistent across the benchmark suites. The PostgreSQL specifics are in this +# file. Results are written as the same benchmark-*.csv the pipeline's benchmark +# analyzer supports. + +# --------------------------------------------------------------------------- +# Configuration (overridable via environment) +# --------------------------------------------------------------------------- +PGBENCH_DURATION="${PGBENCH_DURATION:-240}" +PGBENCH_SCALING_FACTOR="${PGBENCH_SCALING_FACTOR:-100}" + +PGDATA="/tmp/pgdata" +PGPORT=5432 +PGDATABASE="pgbench" +export PGDATA PGPORT + +NUM_CPUS=$(nproc) +HALF_CPUS=$((NUM_CPUS / 2)) +# Single VM with CPU pinning: PostgreSQL on the first half of the cores, +# pgbench client on the second half, to reduce client/server interference. +SERVER_CPUS="0-$((HALF_CPUS - 1))" +CLIENT_CPUS="${HALF_CPUS}-$((NUM_CPUS - 1))" + +# --------------------------------------------------------------------------- +# Kernel management (shared across kernel A/B tests) +# --------------------------------------------------------------------------- +# Sets RESULTS_BUCKET/ARCH/KERNEL_RPM_DIR/KERNEL_FILE, validates the pipeline +# environment, and defines the kernel install/reboot helpers. SOURCE_DIR must +# already be set by the run script before this file is sourced. +source "${SOURCE_DIR}/kernel_helpers.sh" + +# --------------------------------------------------------------------------- +# PostgreSQL / pgbench (test-specific) +# --------------------------------------------------------------------------- +run_as_postgres() +{ + if [ "$(id -u)" -eq 0 ]; then + sudo -u postgres "$@" + else + "$@" + fi +} + +# Shared-buffer size: 25% of RAM, capped at 4GB, floored at 128MB. +get_shared_buffer_size() +{ + local mem_kb buffer_mb + mem_kb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo) + buffer_mb=$((mem_kb / 1024 / 4)) + [ "$buffer_mb" -gt 4096 ] && buffer_mb=4096 + [ "$buffer_mb" -lt 128 ] && buffer_mb=128 + echo "$buffer_mb" +} + +setup_postgresql() +{ + echo "Initializing PostgreSQL database cluster..." + rm -rf "$PGDATA" + mkdir -p "$PGDATA" + + if [ "$(id -u)" -eq 0 ]; then + id postgres &>/dev/null || sudo useradd -r postgres + sudo chown -R postgres:postgres "$PGDATA" + fi + + run_as_postgres /usr/bin/initdb -D "$PGDATA" --encoding=SQL_ASCII --locale=C + + local shared_buffers max_connections + shared_buffers=$(get_shared_buffer_size) + max_connections=$((NUM_CPUS * 4 + 100)) + + cat >>"$PGDATA/postgresql.conf" <"$PGDATA/pg_hba.conf" <>"$PGDATA/logfile" 2>&1 & + + local i + for i in {1..30}; do + sleep 1 + /usr/bin/pg_isready -h localhost -p "$PGPORT" && break + done + if ! /usr/bin/pg_isready -h localhost -p "$PGPORT"; then + echo "ERROR: PostgreSQL failed to start:" >&2 + cat "$PGDATA/logfile" >&2 + return 1 + fi + + run_as_postgres /usr/bin/createdb -h localhost -p "$PGPORT" "$PGDATABASE" + echo "PostgreSQL started" +} + +init_pgbench() +{ + local scaling_factor="${1:-$PGBENCH_SCALING_FACTOR}" + echo "Initializing pgbench tables (scaling factor: $scaling_factor)..." + run_as_postgres /usr/bin/pgbench -h localhost -p "$PGPORT" -i -s "$scaling_factor" "$PGDATABASE" +} + +# Run one pgbench mode (readonly|readwrite) into output_file. +run_pgbench() +{ + local mode="$1" + local output_file="$2" + local duration="${3:-$PGBENCH_DURATION}" + + local clients threads mode_flag="" + [ "$mode" = "readonly" ] && mode_flag="-S" + clients=$((HALF_CPUS * 2)) + threads=$HALF_CPUS + + echo "Running pgbench $mode (clients=$clients, threads=$threads, duration=${duration}s)" + run_as_postgres taskset -c "$CLIENT_CPUS" /usr/bin/pgbench \ + -h localhost -p "$PGPORT" --protocol=prepared \ + -c "$clients" -j "$threads" -T "$duration" -r $mode_flag \ + "$PGDATABASE" >"$output_file" 2>&1 +} + +stop_postgresql() +{ + echo "Stopping PostgreSQL..." + run_as_postgres /usr/bin/pg_ctl -D "$PGDATA" stop -m fast 2>/dev/null || true +} + +# Set up PostgreSQL, run the read-only and read-write benchmarks into +# results_dir, then stop PostgreSQL. Requires at least 4 CPUs. +run_pgbench_suite() +{ + local results_dir="$1" + if [ "$(nproc)" -lt 4 ]; then + echo "ERROR: pgbench benchmark requires at least 4 CPUs" >&2 + return 1 + fi + mkdir -p "$results_dir" + + setup_postgresql + init_pgbench "$PGBENCH_SCALING_FACTOR" + + local mode output + for mode in readonly readwrite; do + echo "=== Running $mode benchmark ===" + output="$results_dir/pgbench_${mode}.txt" + run_pgbench "$mode" "$output" + cat "$output" + done + + stop_postgresql +} + +# Parse pgbench read-only and read-write output into a benchmark CSV that the +# pipeline's benchmark analyzer consumes (same schema as the unixbench test): +# metric,unit,value,more_is_better,kernel_version,instance_id,instance_type,arch +summarize_pgbench_output() +{ + local readonly_file="$1" + local readwrite_file="$2" + local output_csv_file="$3" + + local kernel_version instance_id instance_type arch + kernel_version=$(uname -r) + instance_id=$(ec2-metadata --instance-id 2>/dev/null | cut -d" " -f2 || hostname || echo "unknown") + instance_type=$(ec2-metadata --instance-type 2>/dev/null | cut -d" " -f2 || echo "unknown") + arch=$(uname -m) + + echo "metric,unit,value,more_is_better,kernel_version,instance_id,instance_type,arch" >"$output_csv_file" + + local mode file tps latency + for mode in readonly readwrite; do + [ "$mode" = "readonly" ] && file="$readonly_file" || file="$readwrite_file" + [ -f "$file" ] || { echo "WARNING: $file not found, skipping $mode" >&2; continue; } + + # "tps = NNN (without initial connection time)" / "(excluding connections establishing)" + tps=$(grep "tps = " "$file" | grep -E "(excluding|without)" | awk '{print $3}' | head -1 || true) + # "latency average = NNN ms" + latency=$(grep "latency average" "$file" | awk '{print $4}' | head -1 || true) + + [ -n "$tps" ] && \ + echo "postgresql.${mode}.tps,TPS,${tps},true,${kernel_version},${instance_id},${instance_type},${arch}" >>"$output_csv_file" + [ -n "$latency" ] && \ + echo "postgresql.${mode}.latency_avg,ms,${latency},false,${kernel_version},${instance_id},${instance_type},${arch}" >>"$output_csv_file" + done +} diff --git a/vm-tests/pgbench-kernel-regression/dependencies.txt b/vm-tests/pgbench-kernel-regression/dependencies.txt new file mode 100644 index 0000000..7552a0a --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/dependencies.txt @@ -0,0 +1,5 @@ +# Packages required to run the pgbench PostgreSQL benchmark. +# Provides initdb, postgres, pgbench, pg_isready, createdb and pg_ctl in /usr/bin. +postgresql16-server +postgresql16-contrib +postgresql16 diff --git a/vm-tests/pgbench-kernel-regression/external_requirements.json b/vm-tests/pgbench-kernel-regression/external_requirements.json new file mode 100644 index 0000000..fd217a4 --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/external_requirements.json @@ -0,0 +1,4 @@ +{ + "kernel-rpms/src": false, + "kernel-rpms/binary": true +} diff --git a/vm-tests/pgbench-kernel-regression/kernel_helpers.sh b/vm-tests/pgbench-kernel-regression/kernel_helpers.sh new file mode 120000 index 0000000..31ff984 --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/kernel_helpers.sh @@ -0,0 +1 @@ +../lib/kernel_helpers.sh \ No newline at end of file diff --git a/vm-tests/pgbench-kernel-regression/package_helpers.sh b/vm-tests/pgbench-kernel-regression/package_helpers.sh new file mode 120000 index 0000000..98eee08 --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/package_helpers.sh @@ -0,0 +1 @@ +../lib/package_helpers.sh \ No newline at end of file diff --git a/vm-tests/pgbench-kernel-regression/run-01-setup-kernel-A.sh b/vm-tests/pgbench-kernel-regression/run-01-setup-kernel-A.sh new file mode 100755 index 0000000..db5ef2c --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/run-01-setup-kernel-A.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# +# First run: install PostgreSQL dependencies and the first (lower-version) +# kernel to test. The client reboots into it before run-02. + +set -euxo pipefail + +# Set source directory and source common library for functions and constants +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SOURCE_DIR}/common_lib.sh" + +# Install PostgreSQL packages (from dependencies.txt, not test metadata). +install_test_dependencies + +# Save a build-level identity of the kernel running before we install kernel A. +# get_running_kernel_id combines uname -r + uname -v + the booted vmlinuz hash, +# so it detects a real kernel switch even when two builds share the same NVR +# (e.g. compiler A/B kernels). +kernel_before="$(get_running_kernel_id)" +echo "Kernel before installation: $kernel_before" +save_kernel_version "$kernel_before" "$KERNEL_FILE" + +# Install the kernel with the lower version as the kernel to be used next. +first_kernel=$(get_first_kernel_rpm_from_dir) +install_specified_kernel_rpm "$first_kernel" + +# Stop here; re-execution happens after reboot and continues in run-02-*.sh diff --git a/vm-tests/pgbench-kernel-regression/run-02-run-pgbench-setup-kernel-B.sh b/vm-tests/pgbench-kernel-regression/run-02-run-pgbench-setup-kernel-B.sh new file mode 100755 index 0000000..3b44d7c --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/run-02-run-pgbench-setup-kernel-B.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# +# Second run: run pgbench on the first (base) kernel, then install the second +# (higher-version) kernel to test. + +set -euxo pipefail + +# Set source directory and source common library for functions and constants +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SOURCE_DIR}/common_lib.sh" + +# Confirm the kernel actually changed after the reboot from run-01. Use the +# build-level identity so a same-NVR-but-different-build kernel still counts. +kernel_after="$(get_running_kernel_id)" +echo "Current running kernel: $(uname -r) (id: $kernel_after)" +kernel_before="$(load_kernel_version "$KERNEL_FILE")" +echo "Kernel before installation: $kernel_before" +assert_kernel_changed "$kernel_before" "$kernel_after" +save_kernel_version "$kernel_after" "$KERNEL_FILE" + +# Make sure PostgreSQL is stopped even if the benchmark fails. +trap stop_postgresql EXIT + +# Run pgbench for the base kernel and record the benchmark CSV. +RESULTS_DIR="${PWD}/results" +run_pgbench_suite "$RESULTS_DIR" +summarize_pgbench_output \ + "$RESULTS_DIR/pgbench_readonly.txt" \ + "$RESULTS_DIR/pgbench_readwrite.txt" \ + "benchmark-base-$(uname -r).csv" + +# Install the kernel with the higher version as the kernel to be used next. +last_kernel=$(get_last_kernel_rpm_from_dir) +install_specified_kernel_rpm "$last_kernel" + +# Stop here; re-execution happens after reboot and continues in run-03-*.sh diff --git a/vm-tests/pgbench-kernel-regression/run-03-run-second-pgbench.sh b/vm-tests/pgbench-kernel-regression/run-03-run-second-pgbench.sh new file mode 100755 index 0000000..c8a3798 --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/run-03-run-second-pgbench.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# +# Third run: run pgbench on the second (tip) kernel. + +set -euxo pipefail + +# Set source directory and source common library for functions and constants +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SOURCE_DIR}/common_lib.sh" + +# Confirm the kernel actually changed after the reboot from run-02. +kernel_after="$(get_running_kernel_id)" +echo "Current running kernel: $(uname -r) (id: $kernel_after)" +kernel_before="$(load_kernel_version "$KERNEL_FILE")" +echo "Kernel before installation: $kernel_before" +assert_kernel_changed "$kernel_before" "$kernel_after" +save_kernel_version "$kernel_after" "$KERNEL_FILE" + +# Make sure PostgreSQL is stopped even if the benchmark fails. +trap stop_postgresql EXIT + +# Run pgbench for the tip kernel and record the benchmark CSV. +RESULTS_DIR="${PWD}/results" +run_pgbench_suite "$RESULTS_DIR" +summarize_pgbench_output \ + "$RESULTS_DIR/pgbench_readonly.txt" \ + "$RESULTS_DIR/pgbench_readwrite.txt" \ + "benchmark-tip-$(uname -r).csv" + +# Stop here; this is the last script, no more execution.