diff --git a/.gitignore b/.gitignore index 7ea422a..de34e0c 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ examples/aws/credentials.json # Project-specific config (generated by setup configure) run-config.json demo-config.json +pgbench-config.json # Test kernel RPMs (large binary files) setup/test-kernel-rpms/ diff --git a/README.md b/README.md index 4d08aec..9938b0a 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,13 @@ Using an explicit configuration file (recommended) kernel-ci-cloud-runner aws run --config my-config.json ``` +To also download this run's result files (benchmark CSVs, `result.txt`, console +logs) from S3 to a local directory, pass `--results-dir`; files land under +`DIR//` mirroring the bucket layout: +``` +kernel-ci-cloud-runner aws run --config my-config.json --results-dir ./results +``` + - Check status by pipeline log message: "VMs: X/X spawned, Y successful, 0 failed, 0 missing" - **Logs:** `logs/` @@ -568,7 +575,7 @@ Test: unixbench-kernel-regression [t-test p=0.0000, U-test p=0.0001, Cohen's d=8.13] ------------------------------------------------------------ -Tests with benchmarks: 1 | Regressions found: 1 +Tests with benchmarks: 1 | Regressions found: 1 | Improvements found: 0 Tests with regressions: unixbench-kernel-regression ============================================================ ``` @@ -584,17 +591,20 @@ The `BenchmarkAnalyzer` returns a `PipelineBenchmarkSummary` dataclass with stru The `PipelineBenchmarkSummary` contains: - `test_results` — list of `TestBenchmarkResult`, one per test - `tests_with_regression` / `regression_test_names` — quick summary of which tests regressed +- `tests_with_improvement` / `improvement_test_names` — quick summary of which tests improved Each `TestBenchmarkResult` contains: - `base_kernel` / `tip_kernel` — kernel version strings - `comparisons` — list of `MetricComparison` (one per benchmark metric) - `regressions` — property that filters to only regressed metrics +- `improvements` — property that filters to only improved metrics Each `MetricComparison` contains: - `metric`, `unit`, `more_is_better` — metric identity - `base` / `tip` — `MetricStats` with `mean`, `median`, `stddev`, `cv`, `values` - `pct_change`, `t_pvalue`, `u_pvalue`, `cohens_d` — statistical results - `is_regression` — boolean flag +- `is_improvement` — boolean flag (significant + meaningful change in the better direction) Example integration at the `NOTIFICATION HOOK` in `pipeline.py`: diff --git a/src/kernel_ci_cloud_labs/auth/aws_auth.py b/src/kernel_ci_cloud_labs/auth/aws_auth.py index b502260..046578f 100644 --- a/src/kernel_ci_cloud_labs/auth/aws_auth.py +++ b/src/kernel_ci_cloud_labs/auth/aws_auth.py @@ -285,6 +285,15 @@ def _make_client(service): first_role_arn = next(iter(role_arns.values()), None) task_config["execution_role_arn"] = first_role_arn task_config["task_role_arn"] = first_role_arn + + # Derive a per-run awslogs stream prefix so concurrent runs are + # distinguishable in the shared /ecs/ log group. Prefer + # an explicit config value, else the run's test_id, else "ecs". + if "log_stream_prefix" not in task_config: + test_id = (self.config.get("test_config") or {}).get("test_id") + if test_id: + task_config["log_stream_prefix"] = test_id + logger.debug("Task definition family: %s", task_config.get("family")) logger.debug("Execution role ARN: %s", task_config.get("execution_role_arn")) diff --git a/src/kernel_ci_cloud_labs/auth/aws_task_definition_manager.py b/src/kernel_ci_cloud_labs/auth/aws_task_definition_manager.py index be883d8..246f276 100644 --- a/src/kernel_ci_cloud_labs/auth/aws_task_definition_manager.py +++ b/src/kernel_ci_cloud_labs/auth/aws_task_definition_manager.py @@ -39,12 +39,19 @@ def create(self, resource_name: str, resource_config: Dict[str, Any]) -> str: # Add CloudWatch logs configuration log_group = f"/ecs/{resource_name}" + # The awslogs stream name is "//". + # A per-run prefix (e.g. the run/test id) makes concurrent runs — which + # all log into the same /ecs/ group — easy to tell apart in + # CloudWatch, instead of every task sharing the generic "ecs" prefix. + # The suffix already guarantees stream uniqueness; the prefix + # is purely for human/tool separability. Defaults to "ecs". + stream_prefix = resource_config.get("log_stream_prefix", "ecs") container_def["logConfiguration"] = { "logDriver": "awslogs", "options": { "awslogs-group": log_group, "awslogs-region": resource_config.get("region", "us-west-2"), - "awslogs-stream-prefix": "ecs", + "awslogs-stream-prefix": stream_prefix, "awslogs-create-group": "true", }, } diff --git a/src/kernel_ci_cloud_labs/cli.py b/src/kernel_ci_cloud_labs/cli.py index 76df4ac..4345945 100644 --- a/src/kernel_ci_cloud_labs/cli.py +++ b/src/kernel_ci_cloud_labs/cli.py @@ -1,7 +1,7 @@ """CLI entry point for kernel-ci-cloud-runner. Usage: - kernel-ci-cloud-runner aws run [--config CONFIG] [--config-s3 S3_URI] + kernel-ci-cloud-runner aws run [--config CONFIG] [--config-s3 S3_URI] [--results-dir DIR] kernel-ci-cloud-runner aws analyze --bucket BUCKET --run-prefix PREFIX [--output-dir DIR] kernel-ci-cloud-runner aws setup configure [--prefix PREFIX] [--region REGION] [--output FILE] kernel-ci-cloud-runner aws setup upload-rpms --bucket BUCKET --local-rpms DIR [--region REGION] @@ -79,6 +79,48 @@ def cmd_run(args): run_pipeline(provider, storage, run_dir=run_dir) + # Optionally persist all of this run's S3 result objects to a local + # directory (benchmark CSVs, result.txt, console logs, etc.). aws run + # otherwise only keeps the orchestrator logs under logs/run_*/. + if getattr(args, "results_dir", None): + _download_run_results(storage, args.results_dir, logger) + + +def _download_run_results(storage, results_dir, logger): + """Download every S3 object under this run's prefix into results_dir. + + Preserves the S3 key structure under results_dir// so the + layout matches the bucket. Best-effort: logs and continues on error. + """ + import os as _os + + bucket = getattr(storage, "bucket", None) + run_prefix = getattr(storage, "run_prefix", None) + s3 = getattr(storage, "s3", None) + if not (bucket and run_prefix and s3 is not None): + logger.warning("Cannot download results: bucket/run_prefix/s3 client unavailable") + return + + dest_root = _os.path.join(results_dir, run_prefix) + logger.info("Downloading run results from s3://%s/%s/ to %s", bucket, run_prefix, dest_root) + count = 0 + try: + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket, Prefix=f"{run_prefix}/"): + for obj in page.get("Contents", []): + key = obj["Key"] + if key.endswith("/"): + continue + # Strip the run_prefix so files land under dest_root/. + rel = key[len(run_prefix) + 1:] if key.startswith(run_prefix + "/") else key + local_path = _os.path.join(dest_root, rel) + _os.makedirs(_os.path.dirname(local_path), exist_ok=True) + s3.download_file(bucket, key, local_path) + count += 1 + except Exception as e: # pylint: disable=broad-exception-caught + logger.error("Error downloading run results: %s", e) + logger.info("✓ Downloaded %d result file(s) to %s", count, dest_root) + def cmd_setup_configure(args): """Configure project resources.""" @@ -229,6 +271,11 @@ def main(): "Takes precedence over --config. Designed for EventBridge triggers.", ) run_parser.add_argument("--region", help="AWS region (for S3 config download)") + run_parser.add_argument( + "--results-dir", + help="Download all of this run's S3 result files (benchmark CSVs, " + "result.txt, console logs) into DIR// after the run", + ) run_parser.set_defaults(func=cmd_run) # aws analyze diff --git a/src/kernel_ci_cloud_labs/core/benchmark_analyzer.py b/src/kernel_ci_cloud_labs/core/benchmark_analyzer.py index be7cae5..2e68eff 100644 --- a/src/kernel_ci_cloud_labs/core/benchmark_analyzer.py +++ b/src/kernel_ci_cloud_labs/core/benchmark_analyzer.py @@ -62,12 +62,13 @@ class MetricComparison: u_pvalue: float = 1.0 cohens_d: float = 0.0 is_regression: bool = False + is_improvement: bool = False def __post_init__(self): if self.base.mean != 0: self.pct_change = ((self.tip.mean - self.base.mean) / abs(self.base.mean)) * 100.0 self._compute_tests() - self._detect_regression() + self._classify_change() def _compute_tests(self): """Compute t-test, Mann-Whitney U, and Cohen's d.""" @@ -82,18 +83,27 @@ def _compute_tests(self): # Cohen's d (pooled) self.cohens_d = _cohens_d(base_v, tip_v) - def _detect_regression(self): - """A regression requires significant p-value AND meaningful effect size.""" + def _classify_change(self): + """Classify the change as a regression or an improvement. + + Both require a significant p-value AND a meaningful effect size; they + differ only in direction. A metric moving the "worse" way (down when + more_is_better, up when less_is_better) is a regression; moving the + "better" way is an improvement. Changes that are not both significant + and meaningful are neither (treated as noise). + """ + self.is_regression = False + self.is_improvement = False significant = self.t_pvalue < P_VALUE_THRESHOLD or self.u_pvalue < P_VALUE_THRESHOLD meaningful = abs(self.cohens_d) >= COHENS_D_THRESHOLD if not (significant and meaningful): - self.is_regression = False return - # Direction check: regression means performance got worse - if self.more_is_better: - self.is_regression = self.pct_change < 0 + # pct_change > 0 means tip is larger than base. + got_better = (self.pct_change > 0) if self.more_is_better else (self.pct_change < 0) + if got_better: + self.is_improvement = True else: - self.is_regression = self.pct_change > 0 + self.is_regression = True @dataclass @@ -113,6 +123,14 @@ def regressions(self) -> List[MetricComparison]: def has_regression(self) -> bool: return len(self.regressions) > 0 + @property + def improvements(self) -> List[MetricComparison]: + return [c for c in self.comparisons if c.is_improvement] + + @property + def has_improvement(self) -> bool: + return len(self.improvements) > 0 + @dataclass class PipelineBenchmarkSummary: @@ -125,6 +143,8 @@ class PipelineBenchmarkSummary: failed_test_names: List[str] = field(default_factory=list) tests_with_regression: int = 0 regression_test_names: List[str] = field(default_factory=list) + tests_with_improvement: int = 0 + improvement_test_names: List[str] = field(default_factory=list) class BenchmarkAnalyzer: @@ -159,6 +179,9 @@ def analyze( if result.has_regression: summary.tests_with_regression += 1 summary.regression_test_names.append(test_name) + if result.has_improvement: + summary.tests_with_improvement += 1 + summary.improvement_test_names.append(test_name) return summary @@ -442,15 +465,38 @@ def log_benchmark_summary(summary: PipelineBenchmarkSummary): else: logger.info(" ✓ No regressions detected") + if result.improvements: + logger.info(" ✓ IMPROVEMENTS DETECTED: %d", len(result.improvements)) + for c in result.improvements: + logger.info( + " %s: base=%.2f±%.2f (cv: %.2f) → tip=%.2f±%.2f (cv: %.2f) %s (%+.1f%%) " + "[t-test p=%.4f, U-test p=%.4f, Cohen's d=%.2f]", + c.metric, + c.base.mean, + c.base.stddev, + c.base.cv, + c.tip.mean, + c.tip.stddev, + c.tip.cv, + c.unit, + c.pct_change, + c.t_pvalue, + c.u_pvalue, + c.cohens_d, + ) + logger.info("") logger.info("-" * 60) logger.info( - "Tests with benchmarks: %d | Regressions found: %d", + "Tests with benchmarks: %d | Regressions found: %d | Improvements found: %d", len(summary.test_results), summary.tests_with_regression, + summary.tests_with_improvement, ) if summary.regression_test_names: logger.info("Tests with regressions: %s", ", ".join(summary.regression_test_names)) + if summary.improvement_test_names: + logger.info("Tests with improvements: %s", ", ".join(summary.improvement_test_names)) logger.info("=" * 60) # NOTIFICATION HOOK: Add downstream notifications here, e.g.: diff --git a/src/kernel_ci_cloud_labs/launch_vm.py b/src/kernel_ci_cloud_labs/launch_vm.py index ac4add2..e5b887f 100644 --- a/src/kernel_ci_cloud_labs/launch_vm.py +++ b/src/kernel_ci_cloud_labs/launch_vm.py @@ -8,6 +8,7 @@ import base64 import json import os +import random import shlex import sys import threading @@ -17,6 +18,7 @@ import boto3 from botocore.config import Config +from botocore.exceptions import ClientError from kernel_ci_cloud_labs.core.log_scrub import scrub_text @@ -76,6 +78,57 @@ def log_exception(prefix, exc): sys.stderr.flush() +# AWS error codes that indicate a transient, retryable throttle/capacity +# condition rather than a permanent failure. When many VMs are spawned at +# once (min_count * tests), or several pipeline runs execute concurrently, +# EC2 RunInstances and SSM SendCommand can return these; a bounded retry with +# exponential backoff + jitter smooths the burst out. +_RETRYABLE_ERROR_CODES = frozenset( + { + "RequestLimitExceeded", + "Throttling", + "ThrottlingException", + "ThrottledException", + "TooManyRequestsException", + "RequestThrottled", + "InsufficientInstanceCapacity", + "Unavailable", + "ServiceUnavailable", + "InternalError", + "InternalFailure", + } +) + + +def _call_with_retries(func, *args, description="AWS call", max_attempts=6, base_delay=2.0, **kwargs): + """Call a boto3 operation, retrying transient throttle/capacity errors. + + boto3's adaptive retry mode already handles some throttling, but bursty + RunInstances/SendCommand storms (many VMs, or parallel runs) can still + surface RequestLimitExceeded/InsufficientInstanceCapacity to the caller. + This adds an application-level bounded retry with exponential backoff and + full jitter so a single throttled call doesn't fail an otherwise healthy + VM. Non-retryable ClientErrors are re-raised immediately. + """ + attempt = 0 + while True: + try: + return func(*args, **kwargs) + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + attempt += 1 + if code not in _RETRYABLE_ERROR_CODES or attempt >= max_attempts: + raise + # Exponential backoff with full jitter, capped at 30s. + delay = min(base_delay * (2 ** (attempt - 1)), 30.0) + delay = random.uniform(0, delay) # nosec B311 - jitter, not crypto + log_not( + f"{description}: transient error {code} " + f"(attempt {attempt}/{max_attempts}), retrying in {delay:.1f}s" + ) + time.sleep(delay) + + # Kernel-side fatal/near-fatal markers we scan captured console buffers for. # Hit on any of these gets logged loudly and stamped into the S3 object # metadata so downstream consumers (KCIDB submitter, triage tooling) can @@ -255,7 +308,9 @@ def spawn_vm(self): params["IamInstanceProfile"] = {"Name": self.role_name} log_not("Calling run_instances...") - response = self.ec2.run_instances(**params) + response = _call_with_retries( + self.ec2.run_instances, description="run_instances", **params + ) if not response.get("Instances"): log_error("Failed to launch instance") @@ -321,7 +376,9 @@ def execute_test_via_ssm(self): # pylint: disable=too-many-statements /tmp/test-vm-client.sh {self.s3_bucket} {self.run_prefix} {self.test} {self.max_runtime} """ - response = self.ssm.send_command( + response = _call_with_retries( + self.ssm.send_command, + description="ssm.send_command", InstanceIds=[self.instance_id], DocumentName="AWS-RunShellScript", Parameters={ @@ -762,15 +819,26 @@ def launch_vms_from_config(): # Function to launch and test a single VM instance def launch_and_test_vm(vm_config, instance_num, results_list): - """Launch one VM instance, execute test, and verify results from S3.""" + """Launch one VM instance, execute test, and verify results from S3. + + Any failure here is contained to this one VM: it is recorded as a + failed entry in results_list and never propagates out of the thread, + so one VM's spawn/command/setup error cannot abort the whole run. + """ # Merge shared config with VM-specific config full_vm_config = {**shared_config, **vm_config} vm_name = f"{vm_config.get('instance_type', 'vm')}-{vm_config.get('test', 'test')}-{instance_num}" log_not(f"\n=== Launching VM: {vm_name} ===") - launcher = VMLauncher(full_vm_config) + launcher = None try: + # Construct inside the try: VMLauncher.__init__ resolves the AMI + # via SSM and creates boto3 clients, which can throw (throttle, + # bad config). A failure here must be a failed VM, not an + # unhandled thread death that leaves the VM reported as "missing". + launcher = VMLauncher(full_vm_config) + if not launcher.prepare_test_artifacts(): log_error(f"FAILED: {vm_name} - Could not prepare test artifacts") results_list.append({"vm_name": vm_name, "success": False}) @@ -801,23 +869,33 @@ def launch_and_test_vm(vm_config, instance_num, results_list): log_error(f"FAILED: {vm_name} - Test did not complete successfully") results_list.append({"vm_name": vm_name, "success": False}) - except Exception as e: + except Exception as e: # pylint: disable=broad-exception-caught + # Contain any per-VM error (including VMLauncher construction / + # SSM AMI resolution) so it counts as one failed VM. log_exception(f"FAILED: {vm_name}", e) results_list.append({"vm_name": vm_name, "success": False}) finally: # cleanup() is already internally guarded per-stage, but a thread # that escapes its target with an unhandled exception is silent # by default — wrap once more so any surviving error reaches - # the container log with a traceback. - try: - launcher.cleanup() - except Exception as e: - log_exception(f"cleanup raised for {vm_name}", e) + # the container log with a traceback. Skip if construction failed. + if launcher is not None: + try: + launcher.cleanup() + except Exception as e: # pylint: disable=broad-exception-caught + log_exception(f"cleanup raised for {vm_name}", e) # Launch all VMs in parallel using threads threads = [] results = [] + # Small stagger between thread starts so the initial RunInstances / + # SendCommand burst is spread over time instead of hitting the EC2/SSM + # APIs all in the same instant (which triggers RequestLimitExceeded, + # especially when several pipeline runs execute concurrently). + # Override via PULLAB_VM_SPAWN_STAGGER_SEC; 0 disables it. + spawn_stagger = float(os.getenv("PULLAB_VM_SPAWN_STAGGER_SEC") or 1.0) + for vm_config in expanded_vms: min_count = vm_config.get("min_count", 1) log_not(f"\n=== Queueing {min_count}x {vm_config.get('instance_type')} for test: {vm_config.get('test')} ===") @@ -826,7 +904,9 @@ def launch_and_test_vm(vm_config, instance_num, results_list): for i in range(min_count): thread = threading.Thread(target=launch_and_test_vm, args=(vm_config, i + 1, results)) threads.append(thread) - thread.start() # Start immediately + thread.start() + if spawn_stagger > 0: + time.sleep(spawn_stagger) # Wait for all threads to complete log_not(f"\n=== Waiting for {len(threads)} VMs to complete ===") diff --git a/src/kernel_ci_cloud_labs/providers/aws_provider.py b/src/kernel_ci_cloud_labs/providers/aws_provider.py index 8ae665d..3ef503e 100644 --- a/src/kernel_ci_cloud_labs/providers/aws_provider.py +++ b/src/kernel_ci_cloud_labs/providers/aws_provider.py @@ -11,6 +11,8 @@ import re import time +import botocore.exceptions + from kernel_ci_cloud_labs.core.base_provider import BaseProvider from kernel_ci_cloud_labs.core.logging_config import get_logger from kernel_ci_cloud_labs.core.registry import register_provider @@ -311,7 +313,10 @@ def wait_for_task_completion(self): finishes the kernelci-api node incomplete/Infrastructure with the matched line surfaced in error_msg. * No new VM console output for PULLAB_TASK_HANG_THRESHOLD_SEC seconds - (default 600) -- silent stall, same treatment as a crash. + (default 1200) -- silent stall, same treatment as a crash. The + default accommodates CPU-heavy benchmarks (e.g. UnixBench) whose + console goes quiet for many minutes during a run; lower it via the + env var for faster hang detection on lighter workloads. * Overall PULLAB_TASK_WAIT_TIMEOUT_SEC seconds elapsed (default 3600) -- final safety net for whatever isn't covered above. @@ -331,7 +336,7 @@ def wait_for_task_completion(self): poll_interval = float(os.getenv("PULLAB_TASK_POLL_INTERVAL_SEC") or 30) log_interval = float(os.getenv("PULLAB_TASK_PROGRESS_LOG_SEC") or 120) - hang_threshold = float(os.getenv("PULLAB_TASK_HANG_THRESHOLD_SEC") or 600) + hang_threshold = float(os.getenv("PULLAB_TASK_HANG_THRESHOLD_SEC") or 1200) overall_timeout = float(os.getenv("PULLAB_TASK_WAIT_TIMEOUT_SEC") or 3600) start = time.time() @@ -375,9 +380,31 @@ def wait_for_task_completion(self): # Tail the VM console group for crash patterns / progress. if cw_manager is not None: - new_events = cw_manager.get_logs_with_filter( - start_time=last_event_ms + 1 - ) or [] + log_fetch_ok = True + try: + new_events = cw_manager.get_logs_with_filter( + start_time=last_event_ms + 1 + ) or [] + except botocore.exceptions.ClientError as e: + # A transient failure retrieving logs (most commonly an + # ExpiredTokenException while the credential provider is + # mid-refresh) must NOT be treated as "no console output": + # otherwise the hang timer keeps advancing and could + # false-positive kill a healthy run. Refresh the logs + # client, skip the hang check this cycle, and retry next + # poll once fresh credentials are resolved. + log_fetch_ok = False + new_events = [] + code = e.response.get("Error", {}).get("Code", "") + logger.warning( + "Transient error tailing VM logs (%s); refreshing logs " + "client and pausing hang detection this cycle", code or e, + ) + try: + cw_manager.client = self.auth.get_client("logs") + except Exception as refresh_e: # pylint: disable=broad-exception-caught + logger.warning("Could not refresh logs client: %s", refresh_e) + if new_events: last_event_seen_at = time.time() for ev in new_events: @@ -393,6 +420,10 @@ def wait_for_task_completion(self): ) self.terminate_container() raise RuntimeError(f"kernel crash detected in VM: {msg}") + elif not log_fetch_ok: + # Log retrieval failed transiently: don't let the hang timer + # advance across this window. Treat it as "activity seen". + last_event_seen_at = time.time() elif (time.time() - last_event_seen_at) > hang_threshold: logger.error( "No VM console output for %ds (hang threshold %ds) — stopping task", diff --git a/tests/test-in-venv.sh b/tests/test-in-venv.sh index d1d3a1a..9c54c4f 100755 --- a/tests/test-in-venv.sh +++ b/tests/test-in-venv.sh @@ -14,18 +14,27 @@ set -e # Configuration +# PYTHON selects the interpreter used to create the virtual environment. +# Override it to build the venv with a specific version, e.g. +# PYTHON=python3.12 tests/test-in-venv.sh +# It may be a name on PATH or an absolute path. All pip/pytest calls go through +# " -m ..." (never the bare pip/python3 shims). +PYTHON="${PYTHON:-python3}" VENV_DIR=".venv-testing" MODULE_DIR="$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")" STATUS_CACHE="$MODULE_DIR/$VENV_DIR/.git_status_cache" +# Interpreter inside the venv (created from $PYTHON). Used for pip/pytest so the +# correct environment is targeted regardless of which binary bootstrapped it. +VENV_PYTHON="$MODULE_DIR/$VENV_DIR/bin/python" # Function to create and setup virtual environment setup_virtual_environment() { - echo "Setting up virtual environment..." - python3 -m venv "${VENV_DIR}" + echo "Setting up virtual environment with '${PYTHON}'..." + "${PYTHON}" -m venv "${VENV_DIR}" source "${VENV_DIR}/bin/activate" - pip install --upgrade pip - pip install -e ".[dev]" + "${VENV_PYTHON}" -m pip install --upgrade pip + "${VENV_PYTHON}" -m pip install -e ".[dev]" } # Function to activate virtual environment @@ -40,7 +49,7 @@ install_module() { echo "Installing module..." 1>&2 status=0 - output=$(pip install -e "${MODULE_DIR}" 2>&1) || status=$? + output=$("${VENV_PYTHON}" -m pip install -e "${MODULE_DIR}" 2>&1) || status=$? if [ $status -ne 0 ]; then echo "Installation failed, with output:" 1>&2 echo "$output" 1>&2 @@ -53,7 +62,7 @@ run_tests() { echo "Running unit tests..." 1>&2 status=0 - output=$(python3 -m pytest tests/ -v -m "not integration" 2>&1) || status=$? + output=$("${VENV_PYTHON}" -m pytest tests/ -v -m "not integration" 2>&1) || status=$? if [ $status -eq 0 ]; then echo "Unit tests passed" 1>&2 else diff --git a/tests/test_benchmark_analyzer.py b/tests/test_benchmark_analyzer.py new file mode 100644 index 0000000..f6a3409 --- /dev/null +++ b/tests/test_benchmark_analyzer.py @@ -0,0 +1,49 @@ +"""Unit tests for benchmark regression/improvement classification.""" + +__authors__ = ["Norbert Manthey "] +__copyright__ = "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved." +# SPDX-License-Identifier: Apache-2.0 + +from kernel_ci_cloud_labs.core.benchmark_analyzer import MetricComparison, MetricStats + + +def _cmp(base_values, tip_values, unit="lps", more_is_better=True): + return MetricComparison( + metric="test.metric", + unit=unit, + more_is_better=more_is_better, + base=MetricStats(base_values), + tip=MetricStats(tip_values), + ) + + +class TestChangeClassification: + def test_regression_more_is_better_goes_down(self): + # Throughput (more is better) drops significantly -> regression. + c = _cmp([1000, 1005, 995, 1002], [800, 795, 805, 798], more_is_better=True) + assert c.is_regression is True + assert c.is_improvement is False + + def test_improvement_more_is_better_goes_up(self): + # Throughput (more is better) rises significantly -> improvement. + c = _cmp([800, 795, 805, 798], [1000, 1005, 995, 1002], more_is_better=True) + assert c.is_improvement is True + assert c.is_regression is False + + def test_regression_less_is_better_goes_up(self): + # Latency (less is better) rises significantly -> regression. + c = _cmp([10, 10.1, 9.9, 10.0], [20, 20.1, 19.9, 20.0], more_is_better=False) + assert c.is_regression is True + assert c.is_improvement is False + + def test_improvement_less_is_better_goes_down(self): + # Latency (less is better) drops significantly -> improvement. + c = _cmp([20, 20.1, 19.9, 20.0], [10, 10.1, 9.9, 10.0], more_is_better=False) + assert c.is_improvement is True + assert c.is_regression is False + + def test_noise_is_neither(self): + # Tiny change within noise -> neither regression nor improvement. + c = _cmp([1000, 1001, 999, 1000], [1000, 1002, 998, 1001], more_is_better=True) + assert c.is_regression is False + assert c.is_improvement is False diff --git a/tests/test_cli_results_download.py b/tests/test_cli_results_download.py new file mode 100644 index 0000000..22c423d --- /dev/null +++ b/tests/test_cli_results_download.py @@ -0,0 +1,69 @@ +"""Unit tests for the aws-run --results-dir download helper.""" + +__authors__ = ["Norbert Manthey "] +__copyright__ = "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved." +# SPDX-License-Identifier: Apache-2.0 + +import logging +import os +from types import SimpleNamespace +from unittest.mock import Mock + +from kernel_ci_cloud_labs.cli import _download_run_results + +logger = logging.getLogger("test") + + +def _make_storage(keys): + """Build a storage stub whose paginator yields the given S3 keys.""" + s3 = Mock() + paginator = Mock() + paginator.paginate.return_value = [ + {"Contents": [{"Key": k} for k in keys]} + ] + s3.get_paginator.return_value = paginator + + written = {} + + def _download_file(_bucket, key, local_path): + # Record the mapping and create the file so structure can be asserted. + os.makedirs(os.path.dirname(local_path), exist_ok=True) + with open(local_path, "w", encoding="utf-8") as f: + f.write(key) + written[key] = local_path + + s3.download_file.side_effect = _download_file + storage = SimpleNamespace(bucket="results-bkt", run_prefix="run_x_123", s3=s3) + return storage, written + + +def test_download_preserves_structure_under_run_prefix(tmp_path): + keys = [ + "run_x_123/test_pgbench/output/i-1/benchmark-base-k.csv", + "run_x_123/test_pgbench/output/i-1/result.txt", + "run_x_123/summary.json", + ] + storage, written = _make_storage(keys) + _download_run_results(storage, str(tmp_path), logger) + + # Each object lands under //. + assert (tmp_path / "run_x_123" / "test_pgbench" / "output" / "i-1" / "benchmark-base-k.csv").is_file() + assert (tmp_path / "run_x_123" / "test_pgbench" / "output" / "i-1" / "result.txt").is_file() + assert (tmp_path / "run_x_123" / "summary.json").is_file() + assert len(written) == 3 + + +def test_download_skips_folder_placeholder_keys(tmp_path): + keys = ["run_x_123/", "run_x_123/summary.json"] + storage, written = _make_storage(keys) + _download_run_results(storage, str(tmp_path), logger) + # The "directory" key is skipped; only the real object is downloaded. + assert len(written) == 1 + assert (tmp_path / "run_x_123" / "summary.json").is_file() + + +def test_download_noop_when_storage_incomplete(tmp_path): + storage = SimpleNamespace(bucket=None, run_prefix=None, s3=None) + # Should not raise, just warn and return. + _download_run_results(storage, str(tmp_path), logger) + assert not any(tmp_path.iterdir()) diff --git a/tests/test_provider_lifecycle.py b/tests/test_provider_lifecycle.py index 26276aa..e556036 100644 --- a/tests/test_provider_lifecycle.py +++ b/tests/test_provider_lifecycle.py @@ -8,6 +8,7 @@ from types import SimpleNamespace from unittest.mock import Mock, patch +import botocore.exceptions import pytest from kernel_ci_cloud_labs.providers.aws_provider import ( @@ -179,6 +180,30 @@ def test_hang_threshold_terminates_and_raises(self, monkeypatch): p.wait_for_task_completion() mock_ecs.stop_task.assert_called_once() + def test_transient_log_error_does_not_trip_hang_timeout(self, monkeypatch): + # When log retrieval keeps failing (e.g. ExpiredTokenException while + # credentials refresh), the hang timer must NOT advance — the run + # should keep going and only stop on the overall timeout, not the + # (much smaller) hang threshold. + p, mock_ecs = self._make_provider(monkeypatch, env={ + "PULLAB_TASK_POLL_INTERVAL_SEC": "0", + "PULLAB_TASK_HANG_THRESHOLD_SEC": "3", + "PULLAB_TASK_WAIT_TIMEOUT_SEC": "20", + }) + cw_manager = Mock() + expired = botocore.exceptions.ClientError( + {"Error": {"Code": "ExpiredTokenException", "Message": "expired"}}, + "FilterLogEvents", + ) + cw_manager.get_logs_with_filter.side_effect = expired + with patch.object(p, "get_task_status", return_value={"status": "RUNNING", "containers": []}), \ + patch.object(p, "_build_vm_log_manager", return_value=cw_manager): + # Must NOT raise the hang error (3s); must hit the overall timeout (20s). + with pytest.raises(RuntimeError, match="task wait timeout exceeded"): + p.wait_for_task_completion() + # The logs client was refreshed on the transient error. + assert mock_ecs.stop_task.called # terminated on overall timeout + def test_overall_timeout_terminates_and_raises(self, monkeypatch): # No log manager so the only abort path is the overall-timeout cap. p, mock_ecs = self._make_provider(monkeypatch, env={ diff --git a/tests/test_resource_managers.py b/tests/test_resource_managers.py index ff99b11..0ad8d32 100644 --- a/tests/test_resource_managers.py +++ b/tests/test_resource_managers.py @@ -193,3 +193,29 @@ def test_task_definition_includes_cloudwatch_logs(self): assert container_def["logConfiguration"]["logDriver"] == "awslogs" # Log group defaults to /ecs/{family} if not explicitly set assert "/ecs/" in container_def["logConfiguration"]["options"]["awslogs-group"] + # Stream prefix defaults to "ecs" when not configured + assert container_def["logConfiguration"]["options"]["awslogs-stream-prefix"] == "ecs" + + def test_task_definition_custom_log_stream_prefix(self): + """A configured log_stream_prefix is used for the awslogs stream prefix, + so concurrent runs are separable in the shared /ecs/ group.""" + mock_client = Mock() + mock_client.describe_task_definition.side_effect = Exception("Not found") + mock_client.register_task_definition.return_value = { + "taskDefinition": {"taskDefinitionArn": "arn:aws:ecs:us-west-2:123:task-definition/test:1"} + } + + config = { + "family": "test-task", + "container_name": "test-container", + "log_stream_prefix": "gccupdate", + } + + manager = AWSTaskDefinitionManager(mock_client, {}) + manager.ensure_exists("test-task", config) + + container_def = mock_client.register_task_definition.call_args[1]["containerDefinitions"][0] + assert ( + container_def["logConfiguration"]["options"]["awslogs-stream-prefix"] + == "gccupdate" + ) diff --git a/vm-tests/README.md b/vm-tests/README.md index 1180965..725936e 100644 --- a/vm-tests/README.md +++ b/vm-tests/README.md @@ -12,6 +12,7 @@ See the main [README](../README.md) for writing new tests, configuration, and th | `example-kernel-reboot-test` | 3 | yes | Installs two kernels with reboot between each | | `simple-unixbench` | 1 | no | Runs UnixBench on the default kernel | | `unixbench-kernel-regression` | 3 | yes | Installs two kernels, runs UnixBench on each, produces benchmark CSVs | +| `pgbench-kernel-regression` | 3 | yes | Installs two kernels, runs PostgreSQL pgbench (read-only + read-write) on each, produces benchmark CSVs | | `simple-source-reboot` | 2 | yes | Installs kernel from source RPM, reboots, verifies | ## How Multi-Stage Tests Work diff --git a/vm-tests/TODO-shared-lib.md b/vm-tests/TODO-shared-lib.md new file mode 100644 index 0000000..96046b8 --- /dev/null +++ b/vm-tests/TODO-shared-lib.md @@ -0,0 +1,50 @@ +# Shared kernel-management helpers across vm-tests + +## Approach (implemented) + +The kernel install/upgrade helpers live once in: + + vm-tests/lib/kernel_helpers.sh + +Each kernel test includes it with a **symlink** in its own directory: + + vm-tests//kernel_helpers.sh -> ../lib/kernel_helpers.sh + +and its `common_lib.sh` sources it after setting `SOURCE_DIR`: + + source "${SOURCE_DIR}/kernel_helpers.sh" + +Why a symlink works with zero pipeline changes: `upload_test_payload()` builds +the payload with `Path(test_dir).rglob("*")` + `zf.write(...)`, which follows +the symlink and stores the **target's content** as a real file named +`kernel_helpers.sh`. On the VM the payload is extracted flat, so the test dir +gets a normal `kernel_helpers.sh` next to the `run*.sh` scripts. + +Fix once, benefit everywhere: the underscore/dash RPM-version handling, the +FIPS-disable-before-reboot logic, and the `--allowerasing` cross-series install +live only in the shared lib. + +## Status — migration complete + +All kernel tests now source the shared lib and keep only their test-specific +functions in `common_lib.sh`: + +- [x] `pgbench-kernel-regression` — pgbench/PostgreSQL functions local. +- [x] `example-kernel-reboot-test` — no test-specific functions; just sources + the shared lib. +- [x] `simple-source-reboot` — source-RPM build helpers + (`install_source_kernel_rpm`, `build_kernel_rpm_src`, + `get_first_source_kernel_rpm_from_dir`, `install_and_build_kernel`) local. +- [x] `unixbench-kernel-regression` — UnixBench helpers (`prepare_unixbench`, + `run_unixbench`, `summarize_unixbench_log`) local. + +`simple-unixbench` and other non-kernel tests do not install kernels and do not +use the shared lib. + +## Adding a new kernel test + +1. `cd vm-tests/ && ln -s ../lib/kernel_helpers.sh kernel_helpers.sh` +2. In `common_lib.sh`, `source "${SOURCE_DIR}/kernel_helpers.sh"` and add only + test-specific functions. +3. Verify: `bash -n common_lib.sh` and a source-order smoke test with + `SOURCE_DIR` set. diff --git a/vm-tests/example-kernel-reboot-test/common_lib.sh b/vm-tests/example-kernel-reboot-test/common_lib.sh index 20d3868..fa5894b 100644 --- a/vm-tests/example-kernel-reboot-test/common_lib.sh +++ b/vm-tests/example-kernel-reboot-test/common_lib.sh @@ -2,253 +2,11 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -# Common functions for kernel reboot test - -# Get results bucket and test paths from environment -RESULTS_BUCKET="${S3_BUCKET:-}" -ARCH=$(uname -m) -KERNEL_RPM_DIR="/tmp/kernel-rpms" - -# Validate required environment variables -if [ -z "$RESULTS_BUCKET" ] || [ -z "$RUN_PREFIX" ] || [ -z "$TEST_NAME" ]; then - echo "ERROR: Missing required environment variables (S3_BUCKET, RUN_PREFIX, TEST_NAME)" >&2 - exit 1 -fi - -# Error trap handler to show line where error occurred -error_trap() -{ - local exit_code=$? - local line_number=$1 - echo "$(date): ERROR: Script failed at line $line_number with exit code $exit_code" - echo "$(date): ERROR: Command that failed: $(sed -n "${line_number}p" "$0")" - exit $exit_code -} -trap 'error_trap $LINENO' ERR - -#Return current runnning kernel -get_running_kernel() -{ - uname -r -} - -# Install a single given package -install_package() -{ - local pkg="$1" - local output - echo "Installing package $pkg ..." - if output=$(sudo yum install -y "$pkg" 2>&1) || output=$(sudo dnf install -y "$pkg" 2>&1); then - return 0 - else - echo "Failed to install package $pkg:" - echo "$output" - return 1 - fi -} - -# Install all dependencies for this test -install_test_dependencies() -{ - local deps_file="${SOURCE_DIR}/dependencies.txt" - - if [ -f "$deps_file" ]; then - while IFS= read -r pkg || [ -n "$pkg" ]; do - # Skip empty lines and comments - [[ -z "$pkg" || "$pkg" =~ ^[[:space:]]*# ]] && continue - - # Remove leading/trailing whitespace - pkg=$(echo "$pkg" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - - # Install package if not empty - if [ -n "$pkg" ]; then - install_package "$pkg" || return 1 - fi - done <"$deps_file" - else - # Fallback to hardcoded dependencies - install_package gcc make tar || return 1 - fi -} - -# List available kernels from S3 -list_kernels_from_s3() -{ - S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/binary/${ARCH}/" - aws s3 ls "${S3_PATH}" | grep "\.rpm$" | awk '{print $4}' -} - -# Download specific kernel RPM from S3 -download_kernel_rpm() -{ - if [ -z "${1:-}" ]; then - echo "ERROR: download_kernel_rpm requires kernel_name parameter" >&2 - return 1 - fi - local kernel_name="$1" - - S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/binary/${ARCH}/" - - mkdir -p "$KERNEL_RPM_DIR" - local local_path="${KERNEL_RPM_DIR}/${kernel_name}" - - # Download if not already present - if [ -f "$local_path" ]; then - echo "$local_path" - return 0 - fi - - if aws s3 cp "${S3_PATH}${kernel_name}" "$local_path" --no-progress >&2; then - echo "$local_path" - return 0 - else - echo "ERROR: Failed to download kernel" >&2 - return 1 - fi -} - -# Dump boot configuration for debugging kernel install issues -dump_boot_info() -{ - echo "=== Boot Debug Info ===" - echo "--- OS ---" - head -2 /etc/os-release 2>/dev/null || true - echo "--- Running kernel ---" - uname -r - echo "--- Installed kernel packages ---" - rpm -qa 'kernel*' | sort - echo "--- vmlinuz files in /boot ---" - ls -la /boot/vmlinuz-* 2>/dev/null || echo "(none)" - echo "--- BLS entries ---" - ls -la /boot/loader/entries/ 2>/dev/null || echo "(no BLS directory)" - echo "--- grubby default ---" - sudo grubby --default-kernel 2>/dev/null || echo "(grubby --default-kernel failed)" - echo "--- grubby --info=ALL ---" - sudo grubby --info=ALL 2>/dev/null || echo "(grubby --info=ALL failed)" - echo "=== End Boot Debug Info ===" -} - -# Install kernel RPM, make sure it's used as boot target -install_kernel_rpm() -{ - if [ -z "${1:-}" ]; then - echo "ERROR: install_kernel_rpm requires kernel_rpm parameter" >&2 - return 1 - fi - local kernel_rpm="$1" - - # Check architecture compatibility - local host_arch=$(uname -m) - local rpm_arch=$(rpm -qp --queryformat '%{ARCH}' "$kernel_rpm" 2>/dev/null) - - if [ "$rpm_arch" != "$host_arch" ]; then - echo "ERROR: Architecture mismatch - Host: $host_arch, RPM: $rpm_arch" >&2 - return 1 - fi - - echo "kernel before installation: $(uname -r)" - echo "Installing kernel from $kernel_rpm (arch: $rpm_arch)" - - if sudo yum localinstall -y "$kernel_rpm" 2>/dev/null || sudo dnf install -y "$kernel_rpm" 2>/dev/null; then - dump_boot_info - - # Set the newly installed kernel as default boot target. - # Without this, GRUB boots the newest kernel which may not be the one we just installed. - local installed_version - installed_version=$(rpm -qp --queryformat '%{VERSION}' "$kernel_rpm" 2>/dev/null) - - # Find the grubby entry matching the installed kernel version. - # Use grep || true to avoid ERR trap when no match is found. - local grub_kernel - grub_kernel=$(sudo grubby --info=ALL 2>/dev/null \ - | grep "^kernel=" \ - | grep "$installed_version" \ - | head -1 \ - | sed 's/^kernel=//' \ - | tr -d '"' \ - || true) - - if [ -z "$grub_kernel" ]; then - # Upstream make binrpm-pkg kernels don't register with grubby. - # Find the vmlinuz file and add a boot entry manually. - local vmlinuz - vmlinuz=$(ls /boot/vmlinuz-*"$installed_version"* 2>/dev/null | head -1) - if [ -n "$vmlinuz" ]; then - echo "Adding grubby entry for $vmlinuz" - local initrd="/boot/initramfs-${installed_version}.img" - if [ ! -f "$initrd" ]; then - echo "Generating initramfs at $initrd" - sudo dracut --force "$initrd" "$installed_version" 2>/dev/null \ - || sudo mkinitrd "$initrd" "$installed_version" 2>/dev/null \ - || true - fi - if [ -f "$initrd" ]; then - sudo grubby --add-kernel="$vmlinuz" \ - --initrd="$initrd" \ - --title="Linux $installed_version" \ - --copy-default \ - --make-default - echo "✓ Added and set default: $vmlinuz" - else - echo "WARNING: No initramfs for $installed_version, trying set-default anyway" - sudo grubby --set-default="$vmlinuz" || true - fi - grub_kernel="$vmlinuz" - else - echo "WARNING: No vmlinuz found for version $installed_version" - fi - else - echo "Setting default boot kernel to $grub_kernel" - sudo grubby --set-default="$grub_kernel" - fi - - if [ -n "$grub_kernel" ]; then - echo "Verifying default kernel:" - sudo grubby --default-kernel - fi - return 0 - else - echo "ERROR: Failed to install new kernel" >&2 - return 1 - fi -} - -# Return kernel RPM with lowest version (downloads from S3) -get_first_kernel_rpm_from_dir() -{ - local kernels=$(list_kernels_from_s3 | sort -V) - local first_kernel=$(echo "$kernels" | head -n 1) - - if [ -z "$first_kernel" ]; then - return 1 - fi - - download_kernel_rpm "$first_kernel" -} - -# Return kernel RPM with highest version (downloads from S3) -get_last_kernel_rpm_from_dir() -{ - local kernels=$(list_kernels_from_s3 | sort -V) - local last_kernel=$(echo "$kernels" | tail -n 1) - - if [ -z "$last_kernel" ]; then - return 1 - fi - - download_kernel_rpm "$last_kernel" -} - -# Install a given kernel RPM (passed as argument) -install_specified_kernel_rpm() -{ - local kernel_rpm="$1" - - if [ -z "$kernel_rpm" ]; then - echo "ERROR: install_specified_kernel_rpm requires a kernel RPM path" - return 1 - fi - - echo "Installing kernel RPM: $(basename "$kernel_rpm")" - install_kernel_rpm "$kernel_rpm" -} +# Common functions for the kernel reboot test. +# +# All kernel-management logic (environment validation, kernel RPM +# download/selection, install_kernel_rpm, reboot helpers) lives in the shared +# vm-tests/lib/kernel_helpers.sh, included here via the kernel_helpers.sh +# symlink in this directory. SOURCE_DIR is set by the run script before this +# file is sourced. +source "${SOURCE_DIR}/kernel_helpers.sh" diff --git a/vm-tests/example-kernel-reboot-test/kernel_helpers.sh b/vm-tests/example-kernel-reboot-test/kernel_helpers.sh new file mode 120000 index 0000000..31ff984 --- /dev/null +++ b/vm-tests/example-kernel-reboot-test/kernel_helpers.sh @@ -0,0 +1 @@ +../lib/kernel_helpers.sh \ No newline at end of file diff --git a/vm-tests/lib/kernel_helpers.sh b/vm-tests/lib/kernel_helpers.sh new file mode 100644 index 0000000..244ea45 --- /dev/null +++ b/vm-tests/lib/kernel_helpers.sh @@ -0,0 +1,331 @@ +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared kernel-management helpers for vm-tests that install and boot a kernel +# RPM from the pipeline's shared kernel-rpms area. +# +# Sourced by a test's common_lib.sh (which sets SOURCE_DIR first). It packages +# into each test payload via a symlink `kernel_helpers.sh -> ../lib/kernel_helpers.sh` +# in the test directory; the zip step stores the symlink target's content as a +# real file, so on the VM this is a normal file in the flat test dir. +# +# Fix once, benefit everywhere: the underscore/dash RPM-version handling and the +# FIPS-disable-before-reboot logic live here, so all kernel tests share them. + +# --------------------------------------------------------------------------- +# Results bucket and kernel paths from the pipeline environment. +RESULTS_BUCKET="${S3_BUCKET:-}" +ARCH=$(uname -m) +KERNEL_RPM_DIR="/tmp/kernel-rpms" +KERNEL_FILE="${SOURCE_DIR}/kernel_version_before.txt" + +# Validate required environment variables +if [ -z "$RESULTS_BUCKET" ] || [ -z "$RUN_PREFIX" ] || [ -z "$TEST_NAME" ]; then + echo "ERROR: Missing required environment variables (S3_BUCKET, RUN_PREFIX, TEST_NAME)" >&2 + exit 1 +fi + +get_running_kernel() +{ + uname -r +} + +# A build-level fingerprint of the *running* kernel, used to detect that a +# reboot actually switched kernels even when two builds share the same NVR +# (uname -r). +# +# Combines: +# - uname -r : release (NVR); distinguishes normal version bumps. +# - uname -v : build version string, which embeds the build date/time and +# so differs between two builds of the same NVR. +# - sha256 of the booted /boot/vmlinuz- as a strong fallback when +# uname -v happens to match (or is unavailable). +get_running_kernel_id() +{ + local rel ver img_hash="" vmlinuz + rel="$(uname -r)" + ver="$(uname -v)" + vmlinuz="/boot/vmlinuz-${rel}" + if [ -r "$vmlinuz" ] && command -v sha256sum >/dev/null 2>&1; then + img_hash="$(sha256sum "$vmlinuz" 2>/dev/null | awk '{print $1}')" + fi + # Single-line, stable identity string. + echo "${rel}|${ver}|${img_hash}" +} + +save_kernel_version() +{ + local version="$1" + local out_file="$2" + if [ -z "$version" ] || [ -z "$out_file" ]; then + echo "ERROR: save_kernel_version requires version and file" >&2 + return 1 + fi + echo "$version" >"$out_file" +} + +load_kernel_version() +{ + local in_file="$1" + if [ ! -f "$in_file" ]; then + echo "ERROR: Kernel version file not found: $in_file" >&2 + return 1 + fi + cat "$in_file" +} + +assert_kernel_changed() +{ + local before="$1" + local after="$2" + if [ "$before" = "$after" ]; then + echo "ERROR: kernel did not change after reboot (still: $after)" >&2 + echo " If the two kernels share a version-release (NVR) but differ" >&2 + echo " in build (e.g. compiler A/B), ensure run-01 force-reinstalls" >&2 + echo " the RPM and that get_running_kernel_id is used for before/after." >&2 + return 1 + fi + echo "Kernel changed after reboot:" + echo " before: $before" + echo " after: $after" +} + +# List available kernel RPMs from the shared S3 area. +list_kernels_from_s3() +{ + S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/binary/${ARCH}/" + aws s3 ls "${S3_PATH}" | grep "\.rpm$" | awk '{print $4}' +} + +# Download a specific kernel RPM from S3. +download_kernel_rpm() +{ + if [ -z "${1:-}" ]; then + echo "ERROR: download_kernel_rpm requires kernel_name parameter" >&2 + return 1 + fi + local kernel_name="$1" + S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/binary/${ARCH}/" + mkdir -p "$KERNEL_RPM_DIR" + local local_path="${KERNEL_RPM_DIR}/${kernel_name}" + if [ -f "$local_path" ]; then + echo "$local_path" + return 0 + fi + if aws s3 cp "${S3_PATH}${kernel_name}" "$local_path" --no-progress >&2; then + echo "$local_path" + return 0 + else + echo "ERROR: Failed to download kernel" >&2 + return 1 + fi +} + +# Error trap handler to show line where error occurred +error_trap() +{ + local exit_code=$? + local line_number=$1 + echo "$(date): ERROR: Script failed at line $line_number with exit code $exit_code" + echo "$(date): ERROR: Command that failed: $(sed -n "${line_number}p" "$0")" + exit $exit_code +} +trap 'error_trap $LINENO' ERR + +# Install a single given package +install_package() +{ + local pkg="$1" + local output + echo "Installing package $pkg ..." + if output=$(sudo yum install -y "$pkg" 2>&1) || output=$(sudo dnf install -y "$pkg" 2>&1); then + return 0 + else + echo "Failed to install package $pkg:" + echo "$output" + return 1 + fi +} + +# Install all dependencies for this test +install_test_dependencies() +{ + local deps_file="${SOURCE_DIR}/dependencies.txt" + if [ -f "$deps_file" ]; then + while IFS= read -r pkg || [ -n "$pkg" ]; do + [[ -z "$pkg" || "$pkg" =~ ^[[:space:]]*# ]] && continue + pkg=$(echo "$pkg" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -n "$pkg" ] && { install_package "$pkg" || return 1; } + done <"$deps_file" + else + echo "ERROR: dependencies.txt not found" >&2 + return 1 + fi +} + +# List available kernels from S3, dump boot info, install a kernel RPM and make +# it the default boot target. +dump_boot_info() +{ + echo "=== Boot Debug Info ===" + echo "--- OS ---" + head -2 /etc/os-release 2>/dev/null || true + echo "--- Running kernel ---" + uname -r + echo "--- Installed kernel packages ---" + rpm -qa 'kernel*' | sort + echo "--- vmlinuz files in /boot ---" + ls -la /boot/vmlinuz-* 2>/dev/null || echo "(none)" + echo "--- grubby default ---" + sudo grubby --default-kernel 2>/dev/null || echo "(grubby --default-kernel failed)" + echo "=== End Boot Debug Info ===" +} + +install_kernel_rpm() +{ + if [ -z "${1:-}" ]; then + echo "ERROR: install_kernel_rpm requires kernel_rpm parameter" >&2 + return 1 + fi + local kernel_rpm="$1" + + local host_arch=$(uname -m) + local rpm_arch=$(rpm -qp --queryformat '%{ARCH}' "$kernel_rpm" 2>/dev/null) + if [ "$rpm_arch" != "$host_arch" ]; then + echo "ERROR: Architecture mismatch - Host: $host_arch, RPM: $rpm_arch" >&2 + return 1 + fi + + echo "kernel before installation: $(uname -r)" + echo "Installing kernel from $kernel_rpm (arch: $rpm_arch)" + + # Install the kernel RPM. Two wrinkles this must handle: + # + # 1. Same NVR, different build: compiler/optimization A/B kernels can share + # the exact version-release string while carrying different + # code/binaries. A plain "dnf install" of an already-present NVR is a + # no-op ("Nothing to do"), which would leave the old build in place. + # Detect that case and force a reinstall so the new vmlinuz/modules are + # actually written. + # 2. Cross-series conflict: on an AMI whose default kernel is a different + # series (e.g. a 6.18 AMI when installing a 6.1 kernel), the distro + # kernel-tools package conflicts with "kernel-uname-r < ", so a + # plain install is refused; fall back to --allowerasing. + local rpm_nvr + rpm_nvr=$(rpm -qp --queryformat '%{NAME}-%{VERSION}-%{RELEASE}' "$kernel_rpm" 2>/dev/null) + local install_ok=1 + if rpm -q "$rpm_nvr" >/dev/null 2>&1; then + # Same NVR already installed — force reinstall so a different build of + # the same version actually replaces the on-disk kernel image/modules. + echo "Package $rpm_nvr already installed; forcing reinstall (build may differ)" + sudo dnf reinstall -y "$kernel_rpm" 2>/dev/null \ + || sudo rpm -Uvh --force "$kernel_rpm" 2>/dev/null \ + || install_ok=0 + else + sudo dnf install -y "$kernel_rpm" 2>/dev/null \ + || sudo yum localinstall -y "$kernel_rpm" 2>/dev/null \ + || sudo dnf install -y --allowerasing "$kernel_rpm" 2>/dev/null \ + || install_ok=0 + fi + if [ "$install_ok" -eq 1 ]; then + dump_boot_info + local installed_version + installed_version=$(rpm -qp --queryformat '%{VERSION}' "$kernel_rpm" 2>/dev/null) + # RPM VERSION may use underscores (e.g. 6.18.41_nogup) while the kernel + # LOCALVERSION uses dashes (vmlinuz-6.18.41-nogup). Try both variants. + local installed_version_alt="${installed_version//_/-}" + + local grub_kernel + grub_kernel=$(sudo grubby --info=ALL 2>/dev/null \ + | grep "^kernel=" \ + | grep -E "$installed_version|$installed_version_alt" \ + | head -1 \ + | sed 's/^kernel=//' \ + | tr -d '"' \ + || true) + + if [ -z "$grub_kernel" ]; then + local vmlinuz + vmlinuz=$(ls /boot/vmlinuz-*"$installed_version"* 2>/dev/null | head -1) + if [ -z "$vmlinuz" ] && [ "$installed_version_alt" != "$installed_version" ]; then + vmlinuz=$(ls /boot/vmlinuz-*"$installed_version_alt"* 2>/dev/null | head -1) + fi + if [ -n "$vmlinuz" ]; then + echo "Adding grubby entry for $vmlinuz" + # Derive the kernel version from the vmlinuz filename + local kver="${vmlinuz#/boot/vmlinuz-}" + local initrd="/boot/initramfs-${kver}.img" + if [ ! -f "$initrd" ]; then + echo "Generating initramfs at $initrd for kernel $kver" + sudo dracut --force "$initrd" "$kver" 2>/dev/null \ + || sudo mkinitrd "$initrd" "$kver" 2>/dev/null \ + || true + fi + if [ -f "$initrd" ]; then + sudo grubby --add-kernel="$vmlinuz" \ + --initrd="$initrd" \ + --title="Linux $kver" \ + --args="fips=0" \ + --copy-default \ + --make-default + echo "Added and set default: $vmlinuz" + else + echo "WARNING: No initramfs for $kver, trying set-default anyway" + sudo grubby --set-default="$vmlinuz" || true + fi + grub_kernel="$vmlinuz" + else + echo "WARNING: No vmlinuz found for version $installed_version" + fi + else + echo "Setting default boot kernel to $grub_kernel" + sudo grubby --set-default="$grub_kernel" + fi + + if [ -n "$grub_kernel" ]; then + echo "Verifying default kernel:" + sudo grubby --default-kernel + fi + + # Disable FIPS mode system-wide before rebooting into a custom kernel. + # Some AL2023 enable FIPS; unsigned modules (from make binrpm-pkg) + # fail signature verification and cause a kernel panic. + if command -v fips-mode-setup &>/dev/null; then + echo "Disabling FIPS mode for custom kernel boot" + sudo fips-mode-setup --disable 2>/dev/null || true + fi + + return 0 + else + echo "ERROR: Failed to install new kernel" >&2 + return 1 + fi +} + +get_first_kernel_rpm_from_dir() +{ + local kernels=$(list_kernels_from_s3 | sort -V) + local first_kernel=$(echo "$kernels" | head -n 1) + [ -z "$first_kernel" ] && return 1 + download_kernel_rpm "$first_kernel" +} + +get_last_kernel_rpm_from_dir() +{ + local kernels=$(list_kernels_from_s3 | sort -V) + local last_kernel=$(echo "$kernels" | tail -n 1) + [ -z "$last_kernel" ] && return 1 + download_kernel_rpm "$last_kernel" +} + +install_specified_kernel_rpm() +{ + local kernel_rpm="$1" + if [ -z "$kernel_rpm" ]; then + echo "ERROR: install_specified_kernel_rpm requires a kernel RPM path" >&2 + return 1 + fi + echo "Installing kernel RPM: $(basename "$kernel_rpm")" + install_kernel_rpm "$kernel_rpm" +} diff --git a/vm-tests/pgbench-kernel-regression/README.md b/vm-tests/pgbench-kernel-regression/README.md new file mode 100644 index 0000000..3adda28 --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/README.md @@ -0,0 +1,58 @@ +# PostgreSQL pgbench Kernel Regression Test + +Kernel A/B performance regression test using PostgreSQL's `pgbench`. It installs +two kernels in turn on a **single VM** and runs the same read-only and +read-write `pgbench` workload against each, so the pipeline's benchmark analyzer +can flag database-performance regressions between kernel versions. It follows +the same three-stage pattern as `unixbench-kernel-regression`. + +- Dependencies are installed from `dependencies.txt` at run time (not from test + metadata). +- Results are emitted as `benchmark-*.csv` in the schema the pipeline already + supports + +## Test Flow + +1. **run-01-setup-kernel-A.sh** — install PostgreSQL packages, record the running + kernel, install the first (lowest-version) kernel RPM from the shared + `kernel-rpms` area, then reboot. +2. **run-02-run-pgbench-setup-kernel-B.sh** — confirm the kernel changed, run + `pgbench` (read-only + read-write) on the base kernel, then install the second + (highest-version) kernel and reboot. +3. **run-03-run-second-pgbench.sh** — confirm the kernel changed, run `pgbench` + on the tip kernel. + +Single VM, CPU-pinned: PostgreSQL is pinned to the first half of the cores and +the `pgbench` client to the second half, to reduce client/server interference. + +## Output + +- `benchmark-base-.csv` — metrics for the first (base) kernel. +- `benchmark-tip-.csv` — metrics for the second (tip) kernel. + +CSV columns: `metric,unit,value,more_is_better,kernel_version,instance_id,instance_type,arch` + +Metrics captured (per kernel): +- `postgresql.readonly.tps` / `postgresql.readwrite.tps`: transactions/sec (more is better). +- `postgresql.readonly.latency_avg` / `postgresql.readwrite.latency_avg`: average latency in ms (less is better). + +The pipeline's benchmark analyzer compares the base and tip CSVs and reports +regressions. + +## Requirements + +- x86_64 or aarch64 instance with at least 4 vCPUs (CPU pinning splits + server/client across the two halves); `c8i.4xlarge` recommended, `us-west-2` + preferred to raise the chance of landing on the same hardware for both kernels. +- Two kernel RPMs uploaded to the shared kernel-rpms area + (`external_requirements.json` sets `kernel-rpms/binary: true`). The lowest + version becomes the base, the highest becomes the tip. +- System packages from `dependencies.txt` (`postgresql16-server`, + `postgresql16-contrib`, `postgresql16`) are installed in run-01. + +## Configuration (environment overrides) + +| Variable | Default | Purpose | +|---|---|---| +| `PGBENCH_DURATION` | `240` | Duration in seconds of each pgbench run. | +| `PGBENCH_SCALING_FACTOR` | `100` | Database size multiplier passed to `pgbench -i -s`. | diff --git a/vm-tests/pgbench-kernel-regression/common_lib.sh b/vm-tests/pgbench-kernel-regression/common_lib.sh new file mode 100644 index 0000000..4d27ac4 --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/common_lib.sh @@ -0,0 +1,213 @@ +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Common library for the pgbench PostgreSQL kernel A/B benchmark. +# +# The kernel-management half (install first/last kernel from the shared +# kernel-rpms area, reboot between stages, assert the running kernel changed) +# is reused from the unixbench-kernel-regression test so behaviour stays +# consistent across the benchmark suites. The PostgreSQL specifics are in this +# file. Results are written as the same benchmark-*.csv the pipeline's benchmark +# analyzer supports. + +# --------------------------------------------------------------------------- +# Configuration (overridable via environment) +# --------------------------------------------------------------------------- +PGBENCH_DURATION="${PGBENCH_DURATION:-240}" +PGBENCH_SCALING_FACTOR="${PGBENCH_SCALING_FACTOR:-100}" + +PGDATA="/tmp/pgdata" +PGPORT=5432 +PGDATABASE="pgbench" +export PGDATA PGPORT + +NUM_CPUS=$(nproc) +HALF_CPUS=$((NUM_CPUS / 2)) +# Single VM with CPU pinning: PostgreSQL on the first half of the cores, +# pgbench client on the second half, to reduce client/server interference. +SERVER_CPUS="0-$((HALF_CPUS - 1))" +CLIENT_CPUS="${HALF_CPUS}-$((NUM_CPUS - 1))" + +# --------------------------------------------------------------------------- +# Kernel management (shared across kernel A/B tests) +# --------------------------------------------------------------------------- +# Sets RESULTS_BUCKET/ARCH/KERNEL_RPM_DIR/KERNEL_FILE, validates the pipeline +# environment, and defines the kernel install/reboot helpers. SOURCE_DIR must +# already be set by the run script before this file is sourced. +source "${SOURCE_DIR}/kernel_helpers.sh" + +# --------------------------------------------------------------------------- +# PostgreSQL / pgbench (test-specific) +# --------------------------------------------------------------------------- +run_as_postgres() +{ + if [ "$(id -u)" -eq 0 ]; then + sudo -u postgres "$@" + else + "$@" + fi +} + +# Shared-buffer size: 25% of RAM, capped at 4GB, floored at 128MB. +get_shared_buffer_size() +{ + local mem_kb buffer_mb + mem_kb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo) + buffer_mb=$((mem_kb / 1024 / 4)) + [ "$buffer_mb" -gt 4096 ] && buffer_mb=4096 + [ "$buffer_mb" -lt 128 ] && buffer_mb=128 + echo "$buffer_mb" +} + +setup_postgresql() +{ + echo "Initializing PostgreSQL database cluster..." + rm -rf "$PGDATA" + mkdir -p "$PGDATA" + + if [ "$(id -u)" -eq 0 ]; then + id postgres &>/dev/null || sudo useradd -r postgres + sudo chown -R postgres:postgres "$PGDATA" + fi + + run_as_postgres /usr/bin/initdb -D "$PGDATA" --encoding=SQL_ASCII --locale=C + + local shared_buffers max_connections + shared_buffers=$(get_shared_buffer_size) + max_connections=$((NUM_CPUS * 4 + 100)) + + cat >>"$PGDATA/postgresql.conf" <"$PGDATA/pg_hba.conf" <>"$PGDATA/logfile" 2>&1 & + + local i + for i in {1..30}; do + sleep 1 + /usr/bin/pg_isready -h localhost -p "$PGPORT" && break + done + if ! /usr/bin/pg_isready -h localhost -p "$PGPORT"; then + echo "ERROR: PostgreSQL failed to start:" >&2 + cat "$PGDATA/logfile" >&2 + return 1 + fi + + run_as_postgres /usr/bin/createdb -h localhost -p "$PGPORT" "$PGDATABASE" + echo "PostgreSQL started" +} + +init_pgbench() +{ + local scaling_factor="${1:-$PGBENCH_SCALING_FACTOR}" + echo "Initializing pgbench tables (scaling factor: $scaling_factor)..." + run_as_postgres /usr/bin/pgbench -h localhost -p "$PGPORT" -i -s "$scaling_factor" "$PGDATABASE" +} + +# Run one pgbench mode (readonly|readwrite) into output_file. +run_pgbench() +{ + local mode="$1" + local output_file="$2" + local duration="${3:-$PGBENCH_DURATION}" + + local clients threads mode_flag="" + [ "$mode" = "readonly" ] && mode_flag="-S" + clients=$((HALF_CPUS * 2)) + threads=$HALF_CPUS + + echo "Running pgbench $mode (clients=$clients, threads=$threads, duration=${duration}s)" + run_as_postgres taskset -c "$CLIENT_CPUS" /usr/bin/pgbench \ + -h localhost -p "$PGPORT" --protocol=prepared \ + -c "$clients" -j "$threads" -T "$duration" -r $mode_flag \ + "$PGDATABASE" >"$output_file" 2>&1 +} + +stop_postgresql() +{ + echo "Stopping PostgreSQL..." + run_as_postgres /usr/bin/pg_ctl -D "$PGDATA" stop -m fast 2>/dev/null || true +} + +# Set up PostgreSQL, run the read-only and read-write benchmarks into +# results_dir, then stop PostgreSQL. Requires at least 4 CPUs. +run_pgbench_suite() +{ + local results_dir="$1" + if [ "$(nproc)" -lt 4 ]; then + echo "ERROR: pgbench benchmark requires at least 4 CPUs" >&2 + return 1 + fi + mkdir -p "$results_dir" + + setup_postgresql + init_pgbench "$PGBENCH_SCALING_FACTOR" + + local mode output + for mode in readonly readwrite; do + echo "=== Running $mode benchmark ===" + output="$results_dir/pgbench_${mode}.txt" + run_pgbench "$mode" "$output" + cat "$output" + done + + stop_postgresql +} + +# Parse pgbench read-only and read-write output into a benchmark CSV that the +# pipeline's benchmark analyzer consumes (same schema as the unixbench test): +# metric,unit,value,more_is_better,kernel_version,instance_id,instance_type,arch +summarize_pgbench_output() +{ + local readonly_file="$1" + local readwrite_file="$2" + local output_csv_file="$3" + + local kernel_version instance_id instance_type arch + kernel_version=$(uname -r) + instance_id=$(ec2-metadata --instance-id 2>/dev/null | cut -d" " -f2 || hostname || echo "unknown") + instance_type=$(ec2-metadata --instance-type 2>/dev/null | cut -d" " -f2 || echo "unknown") + arch=$(uname -m) + + echo "metric,unit,value,more_is_better,kernel_version,instance_id,instance_type,arch" >"$output_csv_file" + + local mode file tps latency + for mode in readonly readwrite; do + [ "$mode" = "readonly" ] && file="$readonly_file" || file="$readwrite_file" + [ -f "$file" ] || { echo "WARNING: $file not found, skipping $mode" >&2; continue; } + + # "tps = NNN (without initial connection time)" / "(excluding connections establishing)" + tps=$(grep "tps = " "$file" | grep -E "(excluding|without)" | awk '{print $3}' | head -1 || true) + # "latency average = NNN ms" + latency=$(grep "latency average" "$file" | awk '{print $4}' | head -1 || true) + + [ -n "$tps" ] && \ + echo "postgresql.${mode}.tps,TPS,${tps},true,${kernel_version},${instance_id},${instance_type},${arch}" >>"$output_csv_file" + [ -n "$latency" ] && \ + echo "postgresql.${mode}.latency_avg,ms,${latency},false,${kernel_version},${instance_id},${instance_type},${arch}" >>"$output_csv_file" + done +} diff --git a/vm-tests/pgbench-kernel-regression/dependencies.txt b/vm-tests/pgbench-kernel-regression/dependencies.txt new file mode 100644 index 0000000..7552a0a --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/dependencies.txt @@ -0,0 +1,5 @@ +# Packages required to run the pgbench PostgreSQL benchmark. +# Provides initdb, postgres, pgbench, pg_isready, createdb and pg_ctl in /usr/bin. +postgresql16-server +postgresql16-contrib +postgresql16 diff --git a/vm-tests/pgbench-kernel-regression/external_requirements.json b/vm-tests/pgbench-kernel-regression/external_requirements.json new file mode 100644 index 0000000..fd217a4 --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/external_requirements.json @@ -0,0 +1,4 @@ +{ + "kernel-rpms/src": false, + "kernel-rpms/binary": true +} diff --git a/vm-tests/pgbench-kernel-regression/kernel_helpers.sh b/vm-tests/pgbench-kernel-regression/kernel_helpers.sh new file mode 120000 index 0000000..31ff984 --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/kernel_helpers.sh @@ -0,0 +1 @@ +../lib/kernel_helpers.sh \ No newline at end of file diff --git a/vm-tests/pgbench-kernel-regression/run-01-setup-kernel-A.sh b/vm-tests/pgbench-kernel-regression/run-01-setup-kernel-A.sh new file mode 100755 index 0000000..db5ef2c --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/run-01-setup-kernel-A.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# +# First run: install PostgreSQL dependencies and the first (lower-version) +# kernel to test. The client reboots into it before run-02. + +set -euxo pipefail + +# Set source directory and source common library for functions and constants +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SOURCE_DIR}/common_lib.sh" + +# Install PostgreSQL packages (from dependencies.txt, not test metadata). +install_test_dependencies + +# Save a build-level identity of the kernel running before we install kernel A. +# get_running_kernel_id combines uname -r + uname -v + the booted vmlinuz hash, +# so it detects a real kernel switch even when two builds share the same NVR +# (e.g. compiler A/B kernels). +kernel_before="$(get_running_kernel_id)" +echo "Kernel before installation: $kernel_before" +save_kernel_version "$kernel_before" "$KERNEL_FILE" + +# Install the kernel with the lower version as the kernel to be used next. +first_kernel=$(get_first_kernel_rpm_from_dir) +install_specified_kernel_rpm "$first_kernel" + +# Stop here; re-execution happens after reboot and continues in run-02-*.sh diff --git a/vm-tests/pgbench-kernel-regression/run-02-run-pgbench-setup-kernel-B.sh b/vm-tests/pgbench-kernel-regression/run-02-run-pgbench-setup-kernel-B.sh new file mode 100755 index 0000000..3b44d7c --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/run-02-run-pgbench-setup-kernel-B.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# +# Second run: run pgbench on the first (base) kernel, then install the second +# (higher-version) kernel to test. + +set -euxo pipefail + +# Set source directory and source common library for functions and constants +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SOURCE_DIR}/common_lib.sh" + +# Confirm the kernel actually changed after the reboot from run-01. Use the +# build-level identity so a same-NVR-but-different-build kernel still counts. +kernel_after="$(get_running_kernel_id)" +echo "Current running kernel: $(uname -r) (id: $kernel_after)" +kernel_before="$(load_kernel_version "$KERNEL_FILE")" +echo "Kernel before installation: $kernel_before" +assert_kernel_changed "$kernel_before" "$kernel_after" +save_kernel_version "$kernel_after" "$KERNEL_FILE" + +# Make sure PostgreSQL is stopped even if the benchmark fails. +trap stop_postgresql EXIT + +# Run pgbench for the base kernel and record the benchmark CSV. +RESULTS_DIR="${PWD}/results" +run_pgbench_suite "$RESULTS_DIR" +summarize_pgbench_output \ + "$RESULTS_DIR/pgbench_readonly.txt" \ + "$RESULTS_DIR/pgbench_readwrite.txt" \ + "benchmark-base-$(uname -r).csv" + +# Install the kernel with the higher version as the kernel to be used next. +last_kernel=$(get_last_kernel_rpm_from_dir) +install_specified_kernel_rpm "$last_kernel" + +# Stop here; re-execution happens after reboot and continues in run-03-*.sh diff --git a/vm-tests/pgbench-kernel-regression/run-03-run-second-pgbench.sh b/vm-tests/pgbench-kernel-regression/run-03-run-second-pgbench.sh new file mode 100755 index 0000000..c8a3798 --- /dev/null +++ b/vm-tests/pgbench-kernel-regression/run-03-run-second-pgbench.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# +# Third run: run pgbench on the second (tip) kernel. + +set -euxo pipefail + +# Set source directory and source common library for functions and constants +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SOURCE_DIR}/common_lib.sh" + +# Confirm the kernel actually changed after the reboot from run-02. +kernel_after="$(get_running_kernel_id)" +echo "Current running kernel: $(uname -r) (id: $kernel_after)" +kernel_before="$(load_kernel_version "$KERNEL_FILE")" +echo "Kernel before installation: $kernel_before" +assert_kernel_changed "$kernel_before" "$kernel_after" +save_kernel_version "$kernel_after" "$KERNEL_FILE" + +# Make sure PostgreSQL is stopped even if the benchmark fails. +trap stop_postgresql EXIT + +# Run pgbench for the tip kernel and record the benchmark CSV. +RESULTS_DIR="${PWD}/results" +run_pgbench_suite "$RESULTS_DIR" +summarize_pgbench_output \ + "$RESULTS_DIR/pgbench_readonly.txt" \ + "$RESULTS_DIR/pgbench_readwrite.txt" \ + "benchmark-tip-$(uname -r).csv" + +# Stop here; this is the last script, no more execution. diff --git a/vm-tests/simple-source-reboot/common_lib.sh b/vm-tests/simple-source-reboot/common_lib.sh index 843816c..3e4cc59 100644 --- a/vm-tests/simple-source-reboot/common_lib.sh +++ b/vm-tests/simple-source-reboot/common_lib.sh @@ -2,112 +2,21 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -KERNEL_BENCH_DIR="kernel-bench" - -# Get results bucket and test paths from environment -RESULTS_BUCKET="${S3_BUCKET:-}" -ARCH=$(uname -m) -KERNEL_RPM_DIR="/tmp/kernel-rpms" -KERNEL_FILE="${SOURCE_DIR}/kernel_version_before.txt" - -# Validate required environment variables -if [ -z "$RESULTS_BUCKET" ] || [ -z "$RUN_PREFIX" ] || [ -z "$TEST_NAME" ]; then - echo "ERROR: Missing required environment variables (S3_BUCKET, RUN_PREFIX, TEST_NAME)" >&2 - exit 1 -fi - -# Error trap handler to show line where error occurred -error_trap() -{ - local exit_code=$? - local line_number=$1 - echo "$(date): ERROR: Script failed at line $line_number with exit code $exit_code" - echo "$(date): ERROR: Command that failed: $(sed -n "${line_number}p" "$0")" - exit $exit_code -} -trap 'error_trap $LINENO' ERR - -get_running_kernel() -{ - uname -r -} - -save_kernel_version() -{ - local version="$1" - local out_file="$2" - - if [ -z "$version" ] || [ -z "$out_file" ]; then - echo "ERROR: save_kernel_version requires version and file" - return 1 - fi - - echo "$version" >"$out_file" -} - -load_kernel_version() -{ - local in_file="$1" - - if [ ! -f "$in_file" ]; then - echo "ERROR: Kernel version file not found: $in_file" - return 1 - fi - - cat "$in_file" -} +# Common library for the simple source-build kernel reboot test. +# +# Binary-kernel install/reboot logic (environment validation, kernel RPM +# download/selection, install_kernel_rpm, reboot helpers) lives in the shared +# vm-tests/lib/kernel_helpers.sh, included via the kernel_helpers.sh symlink in +# this directory. Only the source-RPM build helpers stay here. SOURCE_DIR is +# set by the run script before this file is sourced. -assert_kernel_changed() -{ - local before="$1" - local after="$2" - - if [ "$before" = "$after" ]; then - echo "✗ FAILED: Kernel version did not change (still $after)" - return 1 - fi - - echo "✓ SUCCESS: Kernel version changed from $before to $after" -} - -# Install a single given package -install_package() -{ - local pkg="$1" - local output - echo "Installing package $pkg ..." - if output=$(sudo yum install -y "$pkg" 2>&1) || output=$(sudo dnf install -y "$pkg" 2>&1); then - return 0 - else - echo "Failed to install package $pkg:" - echo "$output" - return 1 - fi -} - -# Install all dependencies for this test -install_test_dependencies() -{ - local deps_file="${SOURCE_DIR}/dependencies.txt" +KERNEL_BENCH_DIR="kernel-bench" - if [ -f "$deps_file" ]; then - while IFS= read -r pkg || [ -n "$pkg" ]; do - # Skip empty lines and comments - [[ -z "$pkg" || "$pkg" =~ ^[[:space:]]*# ]] && continue +source "${SOURCE_DIR}/kernel_helpers.sh" - # Remove leading/trailing whitespace - pkg=$(echo "$pkg" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - - # Install package if not empty - if [ -n "$pkg" ]; then - install_package "$pkg" || return 1 - fi - done <"$deps_file" - else - # Fallback to hardcoded dependencies - install_package gcc make tar || return 1 - fi -} +# --------------------------------------------------------------------------- +# Source-RPM build helpers (specific to this test) +# --------------------------------------------------------------------------- # Install kernel source RPM (extracts source code to ~/rpmbuild/) install_source_kernel_rpm() @@ -168,156 +77,6 @@ build_kernel_rpm_src() fi } -# Dump boot configuration for debugging kernel install issues -dump_boot_info() -{ - echo "=== Boot Debug Info ===" - echo "--- OS ---" - head -2 /etc/os-release 2>/dev/null || true - echo "--- Running kernel ---" - uname -r - echo "--- Installed kernel packages ---" - rpm -qa 'kernel*' | sort - echo "--- vmlinuz files in /boot ---" - ls -la /boot/vmlinuz-* 2>/dev/null || echo "(none)" - echo "--- BLS entries ---" - ls -la /boot/loader/entries/ 2>/dev/null || echo "(no BLS directory)" - echo "--- grubby default ---" - sudo grubby --default-kernel 2>/dev/null || echo "(grubby --default-kernel failed)" - echo "--- grubby --info=ALL ---" - sudo grubby --info=ALL 2>/dev/null || echo "(grubby --info=ALL failed)" - echo "=== End Boot Debug Info ===" -} - -# Install binary kernel RPM -install_kernel_rpm() -{ - if [ -z "${1:-}" ]; then - echo "ERROR: install_kernel_rpm requires kernel_rpm parameter" >&2 - return 1 - fi - local kernel_rpm="$1" - - # Check it's a binary RPM (not source) - if [[ "$kernel_rpm" =~ \.src\.rpm$ ]]; then - echo "ERROR: This is a source RPM, not a binary RPM: $kernel_rpm" >&2 - return 1 - fi - - # Check architecture compatibility - local host_arch=$(uname -m) - local rpm_arch=$(rpm -qp --queryformat '%{ARCH}' "$kernel_rpm" 2>/dev/null) - - if [ "$rpm_arch" != "$host_arch" ]; then - echo "ERROR: Architecture mismatch - Host: $host_arch, RPM: $rpm_arch" >&2 - return 1 - fi - - echo "Installing binary kernel from $kernel_rpm (arch: $rpm_arch)" - - if sudo yum localinstall -y "$kernel_rpm" 2>/dev/null || sudo dnf install -y "$kernel_rpm" 2>/dev/null; then - dump_boot_info - - # Set the newly installed kernel as default boot target. - # Without this, GRUB boots the newest kernel which may not be the one we just installed. - local installed_version - installed_version=$(rpm -qp --queryformat '%{VERSION}' "$kernel_rpm" 2>/dev/null) - - # Find the grubby entry matching the installed kernel version. - # Use grep || true to avoid ERR trap when no match is found. - local grub_kernel - grub_kernel=$(sudo grubby --info=ALL 2>/dev/null \ - | grep "^kernel=" \ - | grep "$installed_version" \ - | head -1 \ - | sed 's/^kernel=//' \ - | tr -d '"' \ - || true) - - if [ -z "$grub_kernel" ]; then - # Upstream make binrpm-pkg kernels don't register with grubby. - # Find the vmlinuz file and add a boot entry manually. - local vmlinuz - vmlinuz=$(ls /boot/vmlinuz-*"$installed_version"* 2>/dev/null | head -1) - if [ -n "$vmlinuz" ]; then - echo "Adding grubby entry for $vmlinuz" - local initrd="/boot/initramfs-${installed_version}.img" - if [ ! -f "$initrd" ]; then - echo "Generating initramfs at $initrd" - sudo dracut --force "$initrd" "$installed_version" 2>/dev/null \ - || sudo mkinitrd "$initrd" "$installed_version" 2>/dev/null \ - || true - fi - if [ -f "$initrd" ]; then - sudo grubby --add-kernel="$vmlinuz" \ - --initrd="$initrd" \ - --title="Linux $installed_version" \ - --copy-default \ - --make-default - echo "✓ Added and set default: $vmlinuz" - else - echo "WARNING: No initramfs for $installed_version, trying set-default anyway" - sudo grubby --set-default="$vmlinuz" || true - fi - grub_kernel="$vmlinuz" - else - echo "WARNING: No vmlinuz found for version $installed_version" - fi - else - echo "Setting default boot kernel to $grub_kernel" - sudo grubby --set-default="$grub_kernel" - fi - - if [ -n "$grub_kernel" ]; then - echo "Verifying default kernel:" - sudo grubby --default-kernel - fi - echo "✓ Kernel installed successfully" - return 0 - else - echo "ERROR: Failed to install kernel" >&2 - return 1 - fi -} - -##### GET SRC KERNEL FROM S3 AND DOWNLOAD TO LOCAL MACHINE! -# List available kernels from S3 -list_kernels_from_s3() -{ - S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/src/" - aws s3 ls "${S3_PATH}" | grep "\.rpm$" | awk '{print $4}' -} - -# Download specific kernel RPM from S3 -download_kernel_rpm() -{ - if [ -z "${1:-}" ]; then - echo "ERROR: download_kernel_rpm requires kernel_name parameter" >&2 - return 1 - fi - local kernel_name="$1" - - S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/src/" - - mkdir -p "$KERNEL_RPM_DIR" - local local_path="${KERNEL_RPM_DIR}/${kernel_name}" - - # Download if not already present - if [ -f "$local_path" ]; then - echo "$local_path" - return 0 - fi - - if aws s3 cp "${S3_PATH}${kernel_name}" "$local_path" --no-progress >&2; then - echo "$local_path" - return 0 - else - echo "ERROR: Failed to download kernel" >&2 - return 1 - fi -} - -# Return kernel RPM with lowest version (downloads from S3) get_first_source_kernel_rpm_from_dir() { local kernels=$(list_kernels_from_s3 | sort -V) diff --git a/vm-tests/simple-source-reboot/kernel_helpers.sh b/vm-tests/simple-source-reboot/kernel_helpers.sh new file mode 120000 index 0000000..31ff984 --- /dev/null +++ b/vm-tests/simple-source-reboot/kernel_helpers.sh @@ -0,0 +1 @@ +../lib/kernel_helpers.sh \ No newline at end of file diff --git a/vm-tests/simple-unixbench/common_lib.sh b/vm-tests/simple-unixbench/common_lib.sh index bf437f7..e6afeff 100644 --- a/vm-tests/simple-unixbench/common_lib.sh +++ b/vm-tests/simple-unixbench/common_lib.sh @@ -134,9 +134,9 @@ summarize_unixbench_log() # Parse result lines (first section) - use 6th last as value, 5th last as unit in_results && NF >= 6 { - # Extract metric name (everything except last 5 fields) + # Extract metric name (everything except last 6 fields: value unit (timing info)) metric = "" - for (i = 1; i <= NF-4; i++) { + for (i = 1; i <= NF-6; i++) { if (metric == "") { metric = $i } else { @@ -151,43 +151,14 @@ summarize_unixbench_log() # Clean up metric name gsub(/^\s+|\s+$/, "", metric) - # Determine more_is_better - if (metric ~ /System_Call_Overhead/) { - more_is_better = "false" - } else { - more_is_better = "true" - } - - printf "%s.%s,%s,%s,%s,%s,%s,%s,%s\n", benchmark_version, metric, unit, value, more_is_better, kernel_version, instance_id, instance_type, arch - } - - # Parse index section lines - always use second-to-last column as value - in_index && NF >= 3 && !/BASELINE/ && !/RESULT/ && !/INDEX/ && !/^=/ { - # Extract metric name (everything except last 2 fields) - metric = "" - for (i = 1; i <= NF-3; i++) { - if (metric == "") { - metric = $i - } else { - metric = metric "_" $i - } - } - - # Use second-to-last field as value - value = $(NF-1) - - # Skip lines with "---" values - if (value == "---") { - next - } - - # Clean up metric name - gsub(/^\s+|\s+$/, "", metric) - - unit = "score" + # All UnixBench results in this section are throughput rates + # (lps/lpm/KBps), so higher is better — including "System Call + # Overhead", whose value is syscall round-trips per second (lps). more_is_better = "true" printf "%s.%s,%s,%s,%s,%s,%s,%s,%s\n", benchmark_version, metric, unit, value, more_is_better, kernel_version, instance_id, instance_type, arch } + + # Skip index section entirely - do not parse it ' "$unixbench_log" >>"$output_csv_file" } diff --git a/vm-tests/unixbench-kernel-regression/common_lib.sh b/vm-tests/unixbench-kernel-regression/common_lib.sh index 5e161c8..ef65138 100644 --- a/vm-tests/unixbench-kernel-regression/common_lib.sh +++ b/vm-tests/unixbench-kernel-regression/common_lib.sh @@ -2,152 +2,25 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -# Lib with functions required in multiple test steps +# Common library for the UnixBench kernel A/B regression test. +# +# Kernel-management logic (environment validation, kernel RPM +# download/selection, install_kernel_rpm, reboot helpers) lives in the shared +# vm-tests/lib/kernel_helpers.sh, included via the kernel_helpers.sh symlink in +# this directory. Only the UnixBench-specific pieces stay here. SOURCE_DIR is +# set by the run script before this file is sourced. UNIXBENCH_VERSION=byte-unixbench-6.0.0 UNIXBENCH_TAR_FILE="$UNIXBENCH_VERSION.tar.gz" KERNEL_BENCH_DIR="kernel-bench" -# Get results bucket and test paths from environment -RESULTS_BUCKET="${S3_BUCKET:-}" -ARCH=$(uname -m) -KERNEL_RPM_DIR="/tmp/kernel-rpms" -KERNEL_FILE="${SOURCE_DIR}/kernel_version_before.txt" +source "${SOURCE_DIR}/kernel_helpers.sh" -# Validate required environment variables -if [ -z "$RESULTS_BUCKET" ] || [ -z "$RUN_PREFIX" ] || [ -z "$TEST_NAME" ]; then - echo "ERROR: Missing required environment variables (S3_BUCKET, RUN_PREFIX, TEST_NAME)" >&2 - exit 1 -fi +# --------------------------------------------------------------------------- +# UnixBench-specific helpers +# --------------------------------------------------------------------------- -get_running_kernel() -{ - uname -r -} - -save_kernel_version() -{ - local version="$1" - local out_file="$2" - - if [ -z "$version" ] || [ -z "$out_file" ]; then - echo "ERROR: save_kernel_version requires version and file" - return 1 - fi - - echo "$version" >"$out_file" -} - -load_kernel_version() -{ - local in_file="$1" - - if [ ! -f "$in_file" ]; then - echo "ERROR: Kernel version file not found: $in_file" - return 1 - fi - - cat "$in_file" -} - -assert_kernel_changed() -{ - local before="$1" - local after="$2" - - if [ "$before" = "$after" ]; then - echo "✗ FAILED: Kernel version did not change (still $after)" - return 1 - fi - - echo "✓ SUCCESS: Kernel version changed from $before to $after" -} - -# List available kernels from S3 -list_kernels_from_s3() -{ - S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/binary/${ARCH}/" - aws s3 ls "${S3_PATH}" | grep "\.rpm$" | awk '{print $4}' -} - -# Download specific kernel RPM from S3 -download_kernel_rpm() -{ - if [ -z "${1:-}" ]; then - echo "ERROR: download_kernel_rpm requires kernel_name parameter" >&2 - return 1 - fi - local kernel_name="$1" - - S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/binary/${ARCH}/" - - mkdir -p "$KERNEL_RPM_DIR" - local local_path="${KERNEL_RPM_DIR}/${kernel_name}" - - # Download if not already present - if [ -f "$local_path" ]; then - echo "$local_path" - return 0 - fi - - if aws s3 cp "${S3_PATH}${kernel_name}" "$local_path" --no-progress >&2; then - echo "$local_path" - return 0 - else - echo "ERROR: Failed to download kernel" >&2 - return 1 - fi -} - -# Error trap handler to show line where error occurred -error_trap() -{ - local exit_code=$? - local line_number=$1 - echo "$(date): ERROR: Script failed at line $line_number with exit code $exit_code" - echo "$(date): ERROR: Command that failed: $(sed -n "${line_number}p" "$0")" - exit $exit_code -} -trap 'error_trap $LINENO' ERR - -# Install a single given package -install_package() -{ - local pkg="$1" - local output - echo "Installing package $pkg ..." - if output=$(sudo yum install -y "$pkg" 2>&1) || output=$(sudo dnf install -y "$pkg" 2>&1); then - return 0 - else - echo "Failed to install package $pkg:" - echo "$output" - return 1 - fi -} - -# Install all dependencies for this test -install_test_dependencies() -{ - local deps_file="${SOURCE_DIR}/dependencies.txt" - - if [ -f "$deps_file" ]; then - while IFS= read -r pkg || [ -n "$pkg" ]; do - # Skip empty lines and comments - [[ -z "$pkg" || "$pkg" =~ ^[[:space:]]*# ]] && continue - - # Remove leading/trailing whitespace - pkg=$(echo "$pkg" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - - # Install package if not empty - if [ -n "$pkg" ]; then - install_package "$pkg" || return 1 - fi - done <"$deps_file" - else - # Fallback to hardcoded dependencies - install_package gcc make tar || return 1 - fi -} +# Extract unixbench # Extract unixbench prepare_unixbench() @@ -236,12 +109,10 @@ summarize_unixbench_log() # Clean up metric name gsub(/^\s+|\s+$/, "", metric) - # Determine more_is_better - if (metric ~ /System_Call_Overhead/) { - more_is_better = "false" - } else { - more_is_better = "true" - } + # All first-section UnixBench results are throughput rates (lps/lpm/KBps), + # so higher is better — including "System Call Overhead", whose value is + # syscall round-trips per second (lps), not a time. Do NOT invert it. + more_is_better = "true" printf "%s.%s,%s,%s,%s,%s,%s,%s,%s\n", benchmark_version, metric, unit, value, more_is_better, kernel_version, instance_id, instance_type, arch, arch } @@ -249,156 +120,3 @@ summarize_unixbench_log() # Skip index section entirely - do not parse it ' "$unixbench_log" >>"$output_csv_file" } - -# Dump boot configuration for debugging kernel install issues -dump_boot_info() -{ - echo "=== Boot Debug Info ===" - echo "--- OS ---" - head -2 /etc/os-release 2>/dev/null || true - echo "--- Running kernel ---" - uname -r - echo "--- Installed kernel packages ---" - rpm -qa 'kernel*' | sort - echo "--- vmlinuz files in /boot ---" - ls -la /boot/vmlinuz-* 2>/dev/null || echo "(none)" - echo "--- BLS entries ---" - ls -la /boot/loader/entries/ 2>/dev/null || echo "(no BLS directory)" - echo "--- grubby default ---" - sudo grubby --default-kernel 2>/dev/null || echo "(grubby --default-kernel failed)" - echo "--- grubby --info=ALL ---" - sudo grubby --info=ALL 2>/dev/null || echo "(grubby --info=ALL failed)" - echo "=== End Boot Debug Info ===" -} - -# Install current kernel RPM, make sure it's used as boot target -install_kernel_rpm() -{ - if [ -z "${1:-}" ]; then - echo "ERROR: install_kernel_rpm requires kernel_rpm parameter" >&2 - return 1 - fi - local kernel_rpm="$1" - - # Check architecture compatibility - local host_arch=$(uname -m) - local rpm_arch=$(rpm -qp --queryformat '%{ARCH}' "$kernel_rpm" 2>/dev/null) - - if [ "$rpm_arch" != "$host_arch" ]; then - echo "ERROR: Architecture mismatch - Host: $host_arch, RPM: $rpm_arch" >&2 - return 1 - fi - - echo "kernel before installation: $(uname -r)" - echo "Installing kernel from $kernel_rpm (arch: $rpm_arch)" - - if sudo yum localinstall -y "$kernel_rpm" 2>/dev/null || sudo dnf install -y "$kernel_rpm" 2>/dev/null; then - dump_boot_info - - # Set the newly installed kernel as default boot target. - # Without this, GRUB boots the newest kernel which may not be the one we just installed. - local installed_version - installed_version=$(rpm -qp --queryformat '%{VERSION}' "$kernel_rpm" 2>/dev/null) - - # Find the grubby entry matching the installed kernel version. - # Use grep || true to avoid ERR trap when no match is found. - local grub_kernel - grub_kernel=$(sudo grubby --info=ALL 2>/dev/null \ - | grep "^kernel=" \ - | grep "$installed_version" \ - | head -1 \ - | sed 's/^kernel=//' \ - | tr -d '"' \ - || true) - - if [ -z "$grub_kernel" ]; then - # Upstream make binrpm-pkg kernels don't register with grubby. - # Find the vmlinuz file and add a boot entry manually. - local vmlinuz - vmlinuz=$(ls /boot/vmlinuz-*"$installed_version"* 2>/dev/null | head -1) - if [ -n "$vmlinuz" ]; then - echo "Adding grubby entry for $vmlinuz" - # Copy initrd and args from the current default entry - local default_kernel - default_kernel=$(sudo grubby --default-kernel) - local default_initrd - default_initrd=$(sudo grubby --info="$default_kernel" 2>/dev/null \ - | grep "^initrd=" | sed 's/^initrd=//' | tr -d '"' || true) - local initrd="/boot/initramfs-${installed_version}.img" - # Generate initramfs if it doesn't exist - if [ ! -f "$initrd" ]; then - echo "Generating initramfs at $initrd" - sudo dracut --force "$initrd" "$installed_version" 2>/dev/null \ - || sudo mkinitrd "$initrd" "$installed_version" 2>/dev/null \ - || true - fi - if [ -f "$initrd" ]; then - sudo grubby --add-kernel="$vmlinuz" \ - --initrd="$initrd" \ - --title="Linux $installed_version" \ - --copy-default \ - --make-default - echo "✓ Added and set default: $vmlinuz" - else - echo "WARNING: No initramfs for $installed_version, trying set-default anyway" - sudo grubby --set-default="$vmlinuz" || true - fi - grub_kernel="$vmlinuz" - else - echo "WARNING: No vmlinuz found for version $installed_version" - fi - else - echo "Setting default boot kernel to $grub_kernel" - sudo grubby --set-default="$grub_kernel" - fi - - if [ -n "$grub_kernel" ]; then - echo "Verifying default kernel:" - sudo grubby --default-kernel - fi - return 0 - else - echo "ERROR: Failed to install new kernel" >&2 - return 1 - fi -} - -# Return kernel RPM with lowest version (downloads from S3) -get_first_kernel_rpm_from_dir() -{ - local kernels=$(list_kernels_from_s3 | sort -V) - local first_kernel=$(echo "$kernels" | head -n 1) - - if [ -z "$first_kernel" ]; then - return 1 - fi - - download_kernel_rpm "$first_kernel" -} - -# Return kernel RPM with highest version (downloads from S3) -get_last_kernel_rpm_from_dir() -{ - local kernels=$(list_kernels_from_s3 | sort -V) - local last_kernel=$(echo "$kernels" | tail -n 1) - - if [ -z "$last_kernel" ]; then - return 1 - fi - - download_kernel_rpm "$last_kernel" -} - -# Install a given kernel RPM (passed as argument) -install_specified_kernel_rpm() -{ - local kernel_rpm="$1" - - if [ -z "$kernel_rpm" ]; then - echo "ERROR: install_specified_kernel_rpm requires a kernel RPM path" - return 1 - fi - - echo "Installing kernel RPM: $(basename "$kernel_rpm")" - install_kernel_rpm "$kernel_rpm" -} diff --git a/vm-tests/unixbench-kernel-regression/kernel_helpers.sh b/vm-tests/unixbench-kernel-regression/kernel_helpers.sh new file mode 120000 index 0000000..31ff984 --- /dev/null +++ b/vm-tests/unixbench-kernel-regression/kernel_helpers.sh @@ -0,0 +1 @@ +../lib/kernel_helpers.sh \ No newline at end of file diff --git a/vm-tests/unixbench-kernel-regression/run-01-setup-kernel-A.sh b/vm-tests/unixbench-kernel-regression/run-01-setup-kernel-A.sh index 45b3bfd..6c51974 100755 --- a/vm-tests/unixbench-kernel-regression/run-01-setup-kernel-A.sh +++ b/vm-tests/unixbench-kernel-regression/run-01-setup-kernel-A.sh @@ -18,7 +18,7 @@ install_test_dependencies prepare_unixbench # Save kernel version before -kernel_before="$(get_running_kernel)" +kernel_before="$(get_running_kernel_id)" echo "Kernel before installation: $kernel_before" save_kernel_version "$kernel_before" "$KERNEL_FILE" diff --git a/vm-tests/unixbench-kernel-regression/run-02-run-unixbench-setup-kernel-B.sh b/vm-tests/unixbench-kernel-regression/run-02-run-unixbench-setup-kernel-B.sh index e9d413b..81bc170 100755 --- a/vm-tests/unixbench-kernel-regression/run-02-run-unixbench-setup-kernel-B.sh +++ b/vm-tests/unixbench-kernel-regression/run-02-run-unixbench-setup-kernel-B.sh @@ -14,7 +14,7 @@ SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SOURCE_DIR}/common_lib.sh" #Assert if kernel version has changed -kernel_after="$(get_running_kernel)" +kernel_after="$(get_running_kernel_id)" echo "Current running kernel: $kernel_after" kernel_before="$(load_kernel_version "$KERNEL_FILE")" @@ -27,7 +27,7 @@ save_kernel_version "$kernel_after" "$KERNEL_FILE" RESULTS_DIR="${PWD}/${KERNEL_BENCH_DIR}/first_kernel" mkdir -p "$RESULTS_DIR" run_unixbench "$RESULTS_DIR" -summarize_unixbench_log "$RESULTS_DIR"/unixbench.log "benchmark-base-$(basename $kernel_after).csv" +summarize_unixbench_log "$RESULTS_DIR"/unixbench.log "benchmark-base-$(uname -r).csv" # Install kernel with higher version as kernel to be used next last_kernel=$(get_last_kernel_rpm_from_dir "$KERNEL_RPM_DIR") diff --git a/vm-tests/unixbench-kernel-regression/run-03-run-second-unixbench.sh b/vm-tests/unixbench-kernel-regression/run-03-run-second-unixbench.sh index 242f976..e5ebe41 100755 --- a/vm-tests/unixbench-kernel-regression/run-03-run-second-unixbench.sh +++ b/vm-tests/unixbench-kernel-regression/run-03-run-second-unixbench.sh @@ -14,7 +14,7 @@ SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SOURCE_DIR}/common_lib.sh" #Assert if kernel version has changed -kernel_after="$(get_running_kernel)" +kernel_after="$(get_running_kernel_id)" echo "Current running kernel: $kernel_after" kernel_before="$(load_kernel_version "$KERNEL_FILE")" @@ -27,6 +27,6 @@ save_kernel_version "$kernel_after" "$KERNEL_FILE" RESULTS_DIR="${PWD}/${KERNEL_BENCH_DIR}/last_kernel" mkdir -p "$RESULTS_DIR" run_unixbench "$RESULTS_DIR" -summarize_unixbench_log "$RESULTS_DIR"/unixbench.log "benchmark-tip-$(basename $kernel_after).csv" +summarize_unixbench_log "$RESULTS_DIR"/unixbench.log "benchmark-tip-$(uname -r).csv" # Stop here, this is the last script, no more execution