diff --git a/test/big-tests/aws/.gitignore b/test/big-tests/aws/.gitignore new file mode 100644 index 00000000..a43e5da9 --- /dev/null +++ b/test/big-tests/aws/.gitignore @@ -0,0 +1,5 @@ +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars +crash.log diff --git a/test/big-tests/aws/.terraform.lock.hcl b/test/big-tests/aws/.terraform.lock.hcl new file mode 100644 index 00000000..39a6fbd3 --- /dev/null +++ b/test/big-tests/aws/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.100.0" + constraints = "~> 5.0" + hashes = [ + "h1:edXOJWE4ORX8Fm+dpVpICzMZJat4AX0VRCAy/xkcOc0=", + "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644", + "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2", + "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274", + "zh:6330766f1d85f01ae6ea90d1b214b8b74cc8c1badc4696b165b36ddd4cc15f7b", + "zh:7c8c2e30d8e55291b86fcb64bdf6c25489d538688545eb48fd74ad622e5d3862", + "zh:99b1003bd9bd32ee323544da897148f46a527f622dc3971af63ea3e251596342", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9f8b909d3ec50ade83c8062290378b1ec553edef6a447c56dadc01a99f4eaa93", + "zh:aaef921ff9aabaf8b1869a86d692ebd24fbd4e12c21205034bb679b9caf883a2", + "zh:ac882313207aba00dd5a76dbd572a0ddc818bb9cbf5c9d61b28fe30efaec951e", + "zh:bb64e8aff37becab373a1a0cc1080990785304141af42ed6aa3dd4913b000421", + "zh:dfe495f6621df5540d9c92ad40b8067376350b005c637ea6efac5dc15028add4", + "zh:f0ddf0eaf052766cfe09dea8200a946519f653c384ab4336e2a4a64fdd6310e9", + "zh:f1b7e684f4c7ae1eed272b6de7d2049bb87a0275cb04dbb7cda6636f600699c9", + "zh:ff461571e3f233699bf690db319dfe46aec75e58726636a0d97dd9ac6e32fb70", + ] +} diff --git a/test/big-tests/aws/AGENTS.md b/test/big-tests/aws/AGENTS.md new file mode 120000 index 00000000..42061c01 --- /dev/null +++ b/test/big-tests/aws/AGENTS.md @@ -0,0 +1 @@ +README.md \ No newline at end of file diff --git a/test/big-tests/aws/README.md b/test/big-tests/aws/README.md new file mode 100644 index 00000000..0a6ea246 --- /dev/null +++ b/test/big-tests/aws/README.md @@ -0,0 +1,154 @@ +# Big tests on one EC2 instance + +After each use, update this runbook with what will make future runs smoother and +repeatable. Keep it concise and broadly applicable; avoid unnecessary detail +and instructions that depend on incidental choices from a particular run. + +This Terraform configuration runs the [big-test WDL](../genomicsqlite_big_tests.wdl) +with miniwdl on an on-demand `c7gd.4xlarge` in the default VPC. The instance +has 16 Arm64 vCPUs, 32 GiB RAM, a 950 GB NVMe instance store, and a separate +50 GiB encrypted gp3 root volume. The default region is `us-west-2`. + +Allow several hours and at least 550 GiB of scratch space for the default +inputs, which occupy about 33 GiB; actual scratch usage can be higher, +especially during the SAM QNAME sort. The WDL requests 16 CPUs per task, so +miniwdl runs one at a time, although a task may use fewer CPUs. + +## Deploy and connect + +The configuration uses Canonical's public Ubuntu 26.04 Arm64 AMI, a default +VPC and subnet, internet access and a public IPv4 address. It creates an +outbound-only security group and an instance role for Session Manager and +CloudWatch metrics; it does not require SSH keys or account-specific IDs. The +caller needs Terraform's EC2 and IAM permissions (including `iam:PassRole`), +Session Manager permission, and the AWS CLI Session Manager plugin. Set +`region` or `availability_zone` with Terraform variables if needed. + +From the repository root: + +```sh +cd test/big-tests/aws +terraform init +terraform plan +terraform apply +# Run the command printed by: +terraform output -raw session_command +``` + +On the instance, wait for bootstrap and enter a fresh Ubuntu login so Docker +group membership takes effect: + +```sh +cloud-init status --wait +sudo -iu ubuntu +findmnt /scratch +df -h /scratch +docker info --format '{{.DockerRootDir}}' +grep '^root' /etc/containerd/config.toml +miniwdl --version +``` + +The instance-store device should be mounted at `/scratch`, with Docker at +`/scratch/docker` and containerd at `/scratch/containerd`. The scratch setup +formats a blank instance store, preserves its filesystem on reboot, and refuses +unexpected disk layouts. Docker and containerd depend on that setup. If boot +fails, inspect `journalctl -u genomicsqlite-scratch`, +`journalctl -u genomicsqlite-downloader`, and `/var/log/cloud-init-output.log`. +Docker group membership grants root-equivalent access. + +## Monitor in CloudWatch + +Bootstrap installs the CloudWatch agent and publishes host-wide CPU, memory, +`/scratch` capacity, and block-device I/O metrics every 10 seconds to the +`GenomicSQLite/BigTests` namespace. In CloudWatch Metrics, select that +namespace, filter by the instance ID printed by Terraform, and graph CPU +`usage_active`/`usage_iowait`, memory `used_percent`/`available_percent`, +`disk_used_percent`/`disk_free` for `/scratch`, and `diskio_*` for the NVMe +instance-store device shown by `findmnt /scratch` and `lsblk`. For disk I/O +byte and time counters, use CloudWatch's **Sum** statistic for each period; +for CPU, memory and capacity use **Average** or **Maximum** as appropriate. +These are host metrics rather than per-task measurements. + +Check `systemctl status amazon-cloudwatch-agent` and the agent log under +`/opt/aws/amazon-cloudwatch-agent/logs/` if metrics do not appear. The +instance role can publish only to this namespace and can describe EC2 tags; +it cannot write CloudWatch logs. The 10-second samples are billed as custom +metrics and may create multiple series for disk devices. CloudWatch keeps +sub-minute data for only three hours, then rolls it up to one-minute data; +export during the run if the full 10-second history matters. The ordinary +`AWS/EC2` CPU metric remains available separately at five-minute resolution. + +For a durable local 10-second copy spanning a multi-hour run, optionally use +`sar -A -o /scratch/miniwdl/sysstat.sa 10` in a detached tmux session (install +`sysstat` if needed). Stop it with Ctrl-C after the run. Its raw file includes +the hostname and must be sanitized before publication. + +## Run + +Choose the WDL revision and source revision deliberately. They are separate: +the WDL's `git_revision` input selects the GenomicSQLite source to build and +defaults to the moving `main` branch. The build task clones +`https://github.com/mlin/GenomicSQLite.git`, so the chosen source commit must +be available there. The following example resolves `main` to a commit; fetch +the WDL revision from its source if it is not in the clone. + +```sh +git clone https://github.com/mlin/GenomicSQLite.git +cd GenomicSQLite +# Fetch if necessary, then check out the desired WDL revision. +git rev-parse HEAD # record the WDL revision +miniwdl check test/big-tests/genomicsqlite_big_tests.wdl +(cd /scratch/miniwdl && miniwdl run_self_test) +``` + +Run in tmux so an interrupted Session Manager connection does not stop the +workflow. Within the tmux session, record the console output and exit status: + +```sh +tmux new-session -s big-tests +# Inside tmux, from the GenomicSQLite checkout: +set -o pipefail +source_rev=$(git ls-remote origin refs/heads/main | cut -f1) +test -n "$source_rev" || { echo 'Could not resolve source revision' >&2; exit 1; } +printf 'source revision: %s\n' "$source_rev" +miniwdl run --no-color --dir /scratch/miniwdl/runs \ + test/big-tests/genomicsqlite_big_tests.wdl "git_revision=$source_rev" \ + 2>&1 | tee /scratch/miniwdl/big-tests-console.log +printf '%s\n' "${PIPESTATUS[0]}" > /scratch/miniwdl/big-tests.exit +``` + +Detach with `Ctrl-b`, then `d`; reconnect with `tmux attach -t big-tests`. +Monitor `df -h /scratch`, the console log, and the task logs in the run +directory. The workflow prints query results but does not assert their expected +values, so a successful exit is a stability check, not a full correctness +proof. The `.dbi` 404 in the no-index web test is expected fallback behavior. + +miniwdl 1.16.0 uses the Docker backend and the experimental `inlineDockerfile` +extension. Bootstrap builds an Arm64 aria2/AWS downloader image because the +default image is amd64-only; Google Storage uses the multi-architecture Google +Cloud CLI image. Inputs and successful calls are cached under +`/scratch/miniwdl`; call-cache entries reference earlier run directories, so +keep those directories for retries. Cache lookup does not validate mutable +remote URIs or image tags; use immutable revisions or disable cache for a +fresh comparison. The public GCS input and S3 CRAM require internet access; +the CRAM is in `us-east-1`. + +## Preserve results and clean up + +Before teardown, save the workflow and task logs, inputs/outputs, WDL and +source revisions, timings, system configuration, exported CloudWatch metrics, +and a run-directory size inventory. Omit the large databases and inputs if they +are throwaway. A sanitized example is in [runs](../runs/). Check logs +and sysstat data for account IDs, hostnames, network details, and other +identifiers before publishing. The instance role has no S3 upload permissions; +use a separately authorized transfer or add narrowly scoped permissions. + +Instance-store contents—including Docker images, caches, logs, and run +files—are lost on stop, hibernation, termination, or host replacement. Preserve +anything needed **before** stopping the instance. Terraform state is local and +may contain infrastructure details. When finished, from the repository root: + +```sh +cd test/big-tests/aws +terraform destroy +``` diff --git a/test/big-tests/aws/cloudwatch-agent.json b/test/big-tests/aws/cloudwatch-agent.json new file mode 100644 index 00000000..2df9e05d --- /dev/null +++ b/test/big-tests/aws/cloudwatch-agent.json @@ -0,0 +1,28 @@ +{ + "agent": { + "metrics_collection_interval": 10 + }, + "metrics": { + "namespace": "GenomicSQLite/BigTests", + "append_dimensions": { + "InstanceId": "${aws:InstanceId}" + }, + "metrics_collected": { + "cpu": { + "totalcpu": true, + "measurement": ["usage_active", "usage_iowait"] + }, + "mem": { + "measurement": ["used_percent", "available_percent"] + }, + "disk": { + "resources": ["/scratch"], + "measurement": ["used_percent", "free"] + }, + "diskio": { + "resources": ["*"], + "measurement": ["read_bytes", "write_bytes", "io_time"] + } + } + } +} diff --git a/test/big-tests/aws/main.tf b/test/big-tests/aws/main.tf new file mode 100644 index 00000000..4fb060d4 --- /dev/null +++ b/test/big-tests/aws/main.tf @@ -0,0 +1,174 @@ +terraform { + required_version = ">= 1.5.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.region +} + +data "aws_vpc" "default" { + default = true +} + +data "aws_subnets" "default" { + filter { + name = "vpc-id" + values = [data.aws_vpc.default.id] + } + filter { + name = "default-for-az" + values = ["true"] + } + dynamic "filter" { + for_each = var.availability_zone == null ? [] : [var.availability_zone] + content { + name = "availability-zone" + values = [filter.value] + } + } +} + +data "aws_ssm_parameter" "ubuntu_arm64" { + name = "/aws/service/canonical/ubuntu/server/resolute/stable/current/arm64/hvm/ebs-gp3/ami-id" +} + +data "aws_iam_policy_document" "assume_ec2" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "instance" { + name_prefix = "genomicsqlite-big-tests-" + assume_role_policy = data.aws_iam_policy_document.assume_ec2.json +} + +# The minimum AWS-documented permissions for Session Manager. These APIs do +# not support resource-level restrictions; no S3 or Parameter Store access is +# granted to the instance. The person starting a session needs their own IAM +# permission to do so. +data "aws_iam_policy_document" "session_manager" { + statement { + actions = [ + "ssm:UpdateInstanceInformation", + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel", + ] + resources = ["*"] + } +} + +resource "aws_iam_role_policy" "session_manager" { + name = "session-manager" + role = aws_iam_role.instance.id + policy = data.aws_iam_policy_document.session_manager.json +} + +resource "aws_iam_role_policy" "metrics" { + name = "publish-metrics" + role = aws_iam_role.instance.id + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "cloudwatch:PutMetricData" + Resource = "*" + Condition = { + StringEquals = { "cloudwatch:namespace" = "GenomicSQLite/BigTests" } + } + }, + { + Effect = "Allow" + Action = "ec2:DescribeTags" + Resource = "*" + } + ] + }) +} + +resource "aws_iam_instance_profile" "instance" { + name_prefix = "genomicsqlite-big-tests-" + role = aws_iam_role.instance.name +} + +resource "aws_security_group" "instance" { + name_prefix = "genomicsqlite-big-tests-" + description = "Outbound access for GenomicSQLite big tests; access via Session Manager" + vpc_id = data.aws_vpc.default.id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_instance" "runner" { + ami = data.aws_ssm_parameter.ubuntu_arm64.value + instance_type = "c7gd.4xlarge" + subnet_id = sort(data.aws_subnets.default.ids)[0] + associate_public_ip_address = true + vpc_security_group_ids = [aws_security_group.instance.id] + iam_instance_profile = aws_iam_instance_profile.instance.name + + root_block_device { + volume_type = "gp3" + volume_size = 50 + encrypted = true + delete_on_termination = true + } + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + } + + user_data = templatefile("${path.module}/user-data.sh.tftpl", { + scratch_setup = filebase64("${path.module}/scratch-setup.sh") + downloader_image = filebase64("${path.module}/miniwdl-tools.Dockerfile") + metrics_config = filebase64("${path.module}/cloudwatch-agent.json") + }) + user_data_replace_on_change = true + + tags = { + Name = "genomicsqlite-big-tests" + } + + depends_on = [aws_iam_role_policy.session_manager, aws_iam_role_policy.metrics] +} + +variable "region" { + description = "AWS region containing the default VPC and public Ubuntu AMI parameter." + type = string + default = "us-west-2" +} + +variable "availability_zone" { + description = "Optional availability zone for its default subnet; otherwise select a default subnet automatically." + type = string + default = null +} + +output "instance_id" { + value = aws_instance.runner.id +} + +output "session_command" { + description = "Connect using AWS CLI and the Session Manager plugin." + value = "aws ssm start-session --region ${var.region} --target ${aws_instance.runner.id}" +} diff --git a/test/big-tests/aws/miniwdl-tools.Dockerfile b/test/big-tests/aws/miniwdl-tools.Dockerfile new file mode 100644 index 00000000..a8fcdf03 --- /dev/null +++ b/test/big-tests/aws/miniwdl-tools.Dockerfile @@ -0,0 +1,14 @@ +# miniwdl 1.16.0's default tools image is amd64-only. This Arm64 equivalent +# provides the same two programs used by its HTTP/FTP and S3 downloaders. +FROM ubuntu:26.04 AS awscli_builder +RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y curl unzip ca-certificates +RUN curl -fsSL https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip -o /tmp/awscli.zip \ + && unzip -q /tmp/awscli.zip -d /tmp \ + && /tmp/aws/install --install-dir /opt/aws-cli --bin-dir /opt/aws-cli-bin + +FROM ubuntu:26.04 +RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y aria2 ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY --from=awscli_builder /opt/aws-cli /opt/aws-cli +COPY --from=awscli_builder /opt/aws-cli-bin/ /usr/local/bin/ +RUN aria2c --version >/dev/null && aws --version diff --git a/test/big-tests/aws/scratch-setup.sh b/test/big-tests/aws/scratch-setup.sh new file mode 100644 index 00000000..f8df509b --- /dev/null +++ b/test/big-tests/aws/scratch-setup.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -euo pipefail + +# NVMe names are not stable: both the EBS root and the instance store use them. +# Refuse anything except exactly one disk identified by its AWS model string. +devices=() +for model_file in /sys/block/nvme*n1/device/model; do + [[ -f "$model_file" ]] || continue + if [[ $(<"$model_file") == "Amazon EC2 NVMe Instance Storage"* ]]; then + disk_dir=$(dirname "$(dirname "$model_file")") + devices+=("/dev/$(basename "$disk_dir")") + fi +done +if (( ${#devices[@]} != 1 )); then + echo "Expected exactly one NVMe instance-store disk; found ${#devices[@]}" >&2 + exit 1 +fi +device=${devices[0]} + +mkdir -p /scratch +if mountpoint -q /scratch; then + [[ $(findmnt -n -o MAJ:MIN --target /scratch) == "$(lsblk -dn -o MAJ:MIN "$device")" ]] +else + # Only format an empty, unpartitioned device. Preserve the filesystem on + # reboot, and refuse unexpected signatures instead of destroying them. + if [[ $(lsblk -nr -o TYPE "$device") != disk ]]; then + echo "Instance-store disk has partitions; refusing to format $device" >&2 + exit 1 + fi + signatures=$(wipefs --noheadings --output TYPE "$device") + if [[ -z "$signatures" ]]; then + mkfs.ext4 -m 0 -L gsql-scratch "$device" + elif [[ "$signatures" != ext4 ]] || [[ $(blkid -s TYPE -o value "$device") != ext4 ]] || + [[ $(blkid -s LABEL -o value "$device") != gsql-scratch ]]; then + echo "Unexpected filesystem or signatures on $device; refusing to format" >&2 + exit 1 + fi + mount "$device" /scratch +fi + +install -d -m 0755 /scratch/docker /scratch/containerd +install -d -o ubuntu -g ubuntu -m 0755 /scratch/miniwdl /scratch/miniwdl/runs \ + /scratch/miniwdl/download-cache /scratch/miniwdl/call-cache diff --git a/test/big-tests/aws/test_scratch_setup.py b/test/big-tests/aws/test_scratch_setup.py new file mode 100644 index 00000000..1fb1346c --- /dev/null +++ b/test/big-tests/aws/test_scratch_setup.py @@ -0,0 +1,101 @@ +"""Exercise scratch-disk safety decisions without touching real block devices.""" + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +MOCK_COMMAND = r"""#!/bin/bash +set -e +case "$(basename "$0")" in + mountpoint) test -e "$TEST_ROOT/mounted" ;; + findmnt) echo 259:1 ;; + lsblk) + if [[ "$*" == *MAJ:MIN* ]]; then echo 259:1 + elif [[ -e "$TEST_ROOT/partitioned" ]]; then printf 'disk\npart\n' + else echo disk; fi ;; + wipefs) + if [[ -e "$TEST_ROOT/badfs" ]]; then echo xfs + elif [[ -e "$TEST_ROOT/fs" ]]; then echo ext4; fi ;; + blkid) if [[ "$*" == *TYPE* ]]; then echo ext4; else echo gsql-scratch; fi ;; + mkfs.ext4) echo format >> "$TEST_ROOT/actions"; touch "$TEST_ROOT/fs" ;; + mount) echo mount >> "$TEST_ROOT/actions"; touch "$TEST_ROOT/mounted" ;; + install) : ;; + *) exit 99 ;; +esac +""" + + +class ScratchSetupTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="gsql-scratch-test-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + bindir = self.root / "bin" + bindir.mkdir() + for name in ( + "mountpoint", + "findmnt", + "lsblk", + "wipefs", + "blkid", + "mkfs.ext4", + "mount", + "install", + ): + command = bindir / name + command.write_text(MOCK_COMMAND) + command.chmod(0o755) + self.model = self.root / "sys/block/nvme1n1/device/model" + self.model.parent.mkdir(parents=True) + self.model.write_text("Amazon EC2 NVMe Instance Storage\n") + source = (Path(__file__).parent / "scratch-setup.sh").read_text() + self.script = self.root / "setup.sh" + self.script.write_text( + source.replace("/sys/block/", str(self.root) + "/sys/block/") + .replace("/dev/", str(self.root) + "/dev/") + .replace("/scratch", str(self.root) + "/scratch") + ) + self.env = dict( + os.environ, TEST_ROOT=str(self.root), PATH=str(bindir) + ":" + os.environ["PATH"] + ) + + def run_setup(self): + return subprocess.run( + ["bash", str(self.script)], check=False, env=self.env, capture_output=True, text=True + ) + + def test_fresh_disk_and_reboot(self): + first = self.run_setup() + self.assertEqual(first.returncode, 0, first.stderr) + (self.root / "mounted").unlink() + reboot = self.run_setup() + self.assertEqual(reboot.returncode, 0, reboot.stderr) + self.assertEqual( + (self.root / "actions").read_text().splitlines(), ["format", "mount", "mount"] + ) + + def test_refuses_unexpected_disk_layout(self): + for marker in ("badfs", "partitioned"): + with self.subTest(marker=marker): + (self.root / marker).touch() + result = self.run_setup() + self.assertNotEqual(result.returncode, 0) + self.assertIn("refusing to format", result.stderr) + self.assertFalse((self.root / "actions").exists()) + (self.root / marker).unlink() + + def test_refuses_ebs_or_multiple_instance_store_disks(self): + self.model.write_text("Amazon Elastic Block Store\n") + self.assertNotEqual(self.run_setup().returncode, 0) + self.model.write_text("Amazon EC2 NVMe Instance Storage\n") + other = self.root / "sys/block/nvme2n1/device/model" + other.parent.mkdir(parents=True) + other.write_text(self.model.read_text()) + self.assertNotEqual(self.run_setup().returncode, 0) + self.assertFalse((self.root / "actions").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/big-tests/aws/user-data.sh.tftpl b/test/big-tests/aws/user-data.sh.tftpl new file mode 100644 index 00000000..009b5636 --- /dev/null +++ b/test/big-tests/aws/user-data.sh.tftpl @@ -0,0 +1,101 @@ +#!/bin/bash +set -euxo pipefail + +install -d -m 0755 /usr/local/sbin /etc/systemd/system/docker.service.d \ + /etc/systemd/system/containerd.service.d /etc/docker /etc/containerd /etc/xdg \ + /opt/genomicsqlite-downloader +base64 -d > /usr/local/sbin/genomicsqlite-scratch <<'SCRATCH_SCRIPT' +${scratch_setup} +SCRATCH_SCRIPT +chmod 0755 /usr/local/sbin/genomicsqlite-scratch +base64 -d > /opt/genomicsqlite-downloader/Dockerfile <<'DOWNLOADER_DOCKERFILE' +${downloader_image} +DOWNLOADER_DOCKERFILE +base64 -d > /etc/amazon-cloudwatch-agent-big-tests.json <<'METRICS_CONFIG' +${metrics_config} +METRICS_CONFIG + +cat > /etc/systemd/system/genomicsqlite-scratch.service <<'UNIT' +[Unit] +Description=Prepare and mount GenomicSQLite instance-store scratch +After=local-fs.target +Before=containerd.service docker.service + +[Service] +Type=oneshot +ExecStartPre=/usr/bin/udevadm settle --timeout=30 +ExecStart=/usr/local/sbin/genomicsqlite-scratch +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +UNIT + +for service in docker containerd; do + cat > "/etc/systemd/system/$service.service.d/scratch.conf" <<'UNIT' +[Unit] +Requires=genomicsqlite-scratch.service +After=genomicsqlite-scratch.service +UNIT +done + +cat > /etc/docker/daemon.json <<'JSON' +{"data-root":"/scratch/docker"} +JSON +cat > /etc/containerd/config.toml <<'TOML' +version = 2 +root = "/scratch/containerd" +TOML +cat > /etc/systemd/system/genomicsqlite-downloader.service <<'UNIT' +[Unit] +Description=Build miniwdl's Arm64 downloader image if needed +Requires=docker.service +After=docker.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/bin/sh -c '/usr/bin/docker image inspect genomicsqlite-miniwdl-tools:arm64 >/dev/null 2>&1 || exec /usr/bin/docker build -t genomicsqlite-miniwdl-tools:arm64 /opt/genomicsqlite-downloader' +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +UNIT +cat > /etc/xdg/miniwdl.cfg <<'CONFIG' +[file_io] +mount_tmpdir = true + +[download_cache] +get = true +put = true +dir = /scratch/miniwdl/download-cache + +[call_cache] +get = true +put = true +dir = /scratch/miniwdl/call-cache + +[download_aria2c] +docker = genomicsqlite-miniwdl-tools:arm64 + +[download_awscli] +docker = genomicsqlite-miniwdl-tools:arm64 + +[download_gsutil] +docker = gcr.io/google.com/cloudsdktool/google-cloud-cli:589.0.0-slim +CONFIG + +systemctl daemon-reload +systemctl enable --now genomicsqlite-scratch.service +apt-get update +DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io python3-venv git curl tmux +curl -fsSL https://amazoncloudwatch-agent.s3.amazonaws.com/ubuntu/arm64/latest/amazon-cloudwatch-agent.deb -o /tmp/amazon-cloudwatch-agent.deb +DEBIAN_FRONTEND=noninteractive apt-get install -y /tmp/amazon-cloudwatch-agent.deb +/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/etc/amazon-cloudwatch-agent-big-tests.json +usermod -aG docker ubuntu +python3 -m venv /opt/miniwdl +/opt/miniwdl/bin/pip install 'miniwdl==1.16.0' +ln -sf /opt/miniwdl/bin/miniwdl /usr/local/bin/miniwdl +systemctl enable --now containerd docker +systemctl daemon-reload +systemctl enable --now genomicsqlite-downloader.service diff --git a/test/genomicsqlite_big_tests.wdl b/test/big-tests/genomicsqlite_big_tests.wdl similarity index 88% rename from test/genomicsqlite_big_tests.wdl rename to test/big-tests/genomicsqlite_big_tests.wdl index 665c7c21..197ccfdb 100644 --- a/test/genomicsqlite_big_tests.wdl +++ b/test/big-tests/genomicsqlite_big_tests.wdl @@ -55,6 +55,7 @@ task build { command <<< set -euxo pipefail + # build libgenomicsqlite.so and loader executables git clone --recursive https://github.com/mlin/GenomicSQLite.git cd GenomicSQLite @@ -79,17 +80,17 @@ task build { } Array[String] apt_deps = [ - "zip", "pigz", "wget", "build-essential", "git-core", "sqlite3", "cmake", "libsqlite3-dev", - "libzstd-dev", "python3-pip", "maven", "cargo", "libhts-dev", "samtools", "tabix", "libcurl4-openssl-dev" + "zip", "pigz", "wget", "build-essential", "git", "sqlite3", "cmake", "libsqlite3-dev", + "libzstd-dev", "python3-pip", "maven", "cargo", "libhts-dev", "samtools", "tabix", "libcurl4-gnutls-dev" ] Array[String] pip_deps = ["pytest", "pytest-xdist"] runtime { - cpu: 8 + cpu: 16 inlineDockerfile: [ - "FROM ubuntu:20.04", + "FROM ubuntu:26.04", "RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ~{sep(' ', apt_deps)}", - "RUN pip3 install ~{sep(' ', pip_deps)}" + "RUN pip3 install --break-system-packages ~{sep(' ', pip_deps)}" ] } } @@ -97,7 +98,7 @@ task build { task test_sam { input { File reads - String cram_ref_fa_gz = "ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/000/001/405/GCA_000001405.15_GRCh38/seqs_for_alignment_pipelines.ucsc_ids/GCA_000001405.15_GRCh38_no_alt_analysis_set.fna.gz" + String cram_ref_fa = "https://s3.amazonaws.com/1000genomes/technical/reference/GRCh38_reference_genome/GRCh38_full_analysis_set_plus_decoy_hla.fa" File genomicsqlite_py File libgenomicsqlite_so @@ -110,7 +111,8 @@ task test_sam { set -euxo pipefail TMPDIR=${TMPDIR:-/tmp} - cp ~{genomicsqlite_py} /usr/lib/python3.8/genomicsqlite.py + GENOMICSQLITE_PY="$PWD/genomicsqlite.py" + cp ~{genomicsqlite_py} "$GENOMICSQLITE_PY" cp ~{libgenomicsqlite_so} /usr/local/lib/libgenomicsqlite.so ldconfig >&2 sha256sum /usr/local/lib/libgenomicsqlite.so @@ -122,7 +124,7 @@ task test_sam { reads_file='~{reads}' if [[ $reads_file == *.cram ]]; then # CRAM given; make BAM to take reference downloads out of the timings - wget -nv -O - '~{cram_ref_fa_gz}' | pigz -dc > "${TMPDIR}/cram_ref.fa" + wget -nv -O "${TMPDIR}/cram_ref.fa" '~{cram_ref_fa}' samtools faidx "${TMPDIR}/cram_ref.fa" bam_file="${TMPDIR}/$(basename "$reads_file" .cram).bam" time samtools view -T "${TMPDIR}/cram_ref.fa" -h -O BAM -@ 8 "$reads_file" > "$bam_file" @@ -135,7 +137,7 @@ task test_sam { >&2 ls -l '~{dbname}' # compaction - time /usr/lib/python3.8/genomicsqlite.py '~{dbname}' --compact --level 9 --inner-page-KiB 64 --outer-page-KiB 1 + time python3 "$GENOMICSQLITE_PY" '~{dbname}' --compact --level 9 --inner-page-KiB 64 --outer-page-KiB 1 >&2 ls -l '~{dbname}'* # GRI query @@ -155,8 +157,7 @@ task test_sam { time sqlite3 '~{dbname}.compact' "SELECT meta1, count(*), avg(length(data)) FROM nested_vfs_zstd_pages GROUP BY meta1" >&2 # add a QNAME-sorted seqs table. TODO: write it into a separate attached db - chmod +x /usr/lib/python3.8/genomicsqlite.py - time /usr/lib/python3.8/genomicsqlite.py '~{dbname}.compact' "PRAGMA journal_mode=off; PRAGMA synchronous=off; CREATE TABLE reads_seqs_by_qname AS SELECT * from reads_seqs NOT INDEXED ORDER BY qname" + time python3 "$GENOMICSQLITE_PY" '~{dbname}.compact' "PRAGMA journal_mode=off; PRAGMA synchronous=off; CREATE TABLE reads_seqs_by_qname AS SELECT * from reads_seqs NOT INDEXED ORDER BY qname" >>> output { @@ -165,12 +166,12 @@ task test_sam { Int reads_original_size = round(size(reads)) } - Array[String] apt_deps = ["sqlite3", "samtools", "tabix", "libzstd1", "pigz", "wget"] + Array[String] apt_deps = ["sqlite3", "python3", "samtools", "tabix", "libzstd1", "pigz", "wget"] runtime { - cpu: 8 + cpu: 16 inlineDockerfile: [ - "FROM ubuntu:20.04", + "FROM ubuntu:26.04", "RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ~{sep(' ', apt_deps)}" ] } @@ -259,12 +260,12 @@ task test_sam_web { killall nginx || true >>> - Array[String] apt_deps = ["sqlite3", "nginx", "openssl", "python3-minimal", "libcurl4", "psmisc"] + Array[String] apt_deps = ["sqlite3", "nginx", "openssl", "python3", "libcurl4", "psmisc"] runtime { - cpu: 8 + cpu: 16 inlineDockerfile: [ - "FROM ubuntu:20.04", + "FROM ubuntu:26.04", "RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ~{sep(' ', apt_deps)}" ] } @@ -284,7 +285,8 @@ task test_vcf { command <<< set -euxo pipefail - cp ~{genomicsqlite_py} /usr/lib/python3.8/genomicsqlite.py + GENOMICSQLITE_PY="$PWD/genomicsqlite.py" + cp ~{genomicsqlite_py} "$GENOMICSQLITE_PY" cp ~{libgenomicsqlite_so} /usr/local/lib/libgenomicsqlite.so ldconfig >&2 sha256sum /usr/local/lib/libgenomicsqlite.so @@ -319,12 +321,12 @@ task test_vcf { Int variants_original_size = round(size(variants)) } - Array[String] apt_deps = ["sqlite3", "samtools", "tabix", "libzstd1", "pigz", "wget"] + Array[String] apt_deps = ["sqlite3", "python3", "samtools", "tabix", "libzstd1", "pigz", "wget"] runtime { - cpu: 8 + cpu: 16 inlineDockerfile: [ - "FROM ubuntu:20.04", + "FROM ubuntu:26.04", "RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ~{sep(' ', apt_deps)}" ] } diff --git a/test/big-tests/runs/big-tests-run-2026-11-01.zip b/test/big-tests/runs/big-tests-run-2026-11-01.zip new file mode 100644 index 00000000..67885a2c Binary files /dev/null and b/test/big-tests/runs/big-tests-run-2026-11-01.zip differ