diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d3c860b..0988a46 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,6 +10,7 @@ on: - ".github/workflows/integration.yml" - "tests/conftest.py" - "pyproject.toml" + - "src/**" permissions: contents: read @@ -66,10 +67,12 @@ jobs: # TODO(#13/#14): ship the suite in the image and exec as an unprivileged user. runner=(docker compose -f "$COMPOSE_FILE" exec -T backend pytest) else - runner=(pixi run -e py313 pytest) + # Independent tests, run side by side (stacks.yml `workers`). + runner=(pixi run -e py313 pytest -n "${{ matrix.workers }}") fi set +e - "${runner[@]}" tests/integration -m "${{ matrix.markers }}" \ + # -rs --no-fold-skipped: name every skipped test and why. + "${runner[@]}" tests/integration -m "${{ matrix.markers }}" -rs --no-fold-skipped \ --stack="${{ matrix.stack }}" --config="${{ matrix.config }}" rc=$? # Exit 5 = no tests selected; tolerated until the contract suite lands. diff --git a/docs/dev/how-to/add-an-integration-stack.md b/docs/dev/how-to/add-an-integration-stack.md index ff9b8e4..e12d990 100644 --- a/docs/dev/how-to/add-an-integration-stack.md +++ b/docs/dev/how-to/add-an-integration-stack.md @@ -24,7 +24,7 @@ You do not need to change any CI workflow. CI finds the stack in `stacks.yml`. ```sh docker compose -f tests/integration/stacks//compose.yml up --wait - pixi run -e py313 pytest tests/integration -m "" --stack= --config=basic + pixi run -e py313 pytest tests/integration -n 4 -m "" --stack= --config=basic pixi run -e py313 pytest tests/integration/test_manifest.py ``` diff --git a/docs/dev/how-to/run-integration-tests-locally.md b/docs/dev/how-to/run-integration-tests-locally.md index 6b720b4..fe00769 100644 --- a/docs/dev/how-to/run-integration-tests-locally.md +++ b/docs/dev/how-to/run-integration-tests-locally.md @@ -16,9 +16,12 @@ CI runs the same commands as below, so you can reproduce any CI job on your mach 2. Run the tests. Copy `` from the stack's entry in `stacks.yml`: ```sh - pixi run -e py313 pytest tests/integration -m "" --stack= --config=basic + pixi run -e py313 pytest tests/integration -n 4 -m "" --stack= --config=basic ``` +`-n 4` runs four tests at the same time, as CI does. Use the stack's `workers` value from +`stacks.yml` if it sets one, or leave `-n` out to run one test at a time while debugging. + For stacks with `exec_in_container: true` (the `local-*` stacks), run the tests inside the container instead: diff --git a/docs/dev/reference/integration-stacks.md b/docs/dev/reference/integration-stacks.md index fa8fb84..db7f286 100644 --- a/docs/dev/reference/integration-stacks.md +++ b/docs/dev/reference/integration-stacks.md @@ -44,6 +44,7 @@ tests/integration/ | `versions` | no | Backend versions to test, for example `[lts, latest]`. Default: `[latest]`. | | `markers` | yes | `pytest -m` expression that selects the tests for this stack. | | `exec_in_container` | no | `true`: run pytest inside the backend container. Default: `false`. | +| `workers` | no | How many tests run at the same time (pytest-xdist). Default: `4`. Set `1` for a backend that cannot take several jobs at once. | ## When a stack gets CI jobs diff --git a/tests/integration/README.md b/tests/integration/README.md index 2bbbadf..6b6a08a 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -5,7 +5,7 @@ called a *stack*. ```sh docker compose -f tests/integration/stacks//compose.yml up --wait -pixi run -e py313 pytest tests/integration -m "" --stack= --config=basic +pixi run -e py313 pytest tests/integration -n 4 -m "" --stack= --config=basic ``` Documentation: diff --git a/tests/integration/generate_matrix.py b/tests/integration/generate_matrix.py index b812193..1e9b83d 100644 --- a/tests/integration/generate_matrix.py +++ b/tests/integration/generate_matrix.py @@ -19,6 +19,10 @@ STACKS_DIR = Path(__file__).parent / "stacks" REQUIRED_KEYS = ("id", "configs", "markers") +# pytest-xdist workers per job. The tests are independent and every backend +# queues concurrent jobs, so they run side by side; a stack whose backend +# cannot take concurrent jobs sets `workers: 1`. +DEFAULT_WORKERS = 4 def load_manifest() -> list[dict[str, Any]]: @@ -42,6 +46,16 @@ def is_ready(entry: dict[str, Any]) -> bool: ).is_dir() +def workers(entry: dict[str, Any]) -> int: + """Return how many pytest-xdist workers the stack's tests run with.""" + value = entry.get("workers", DEFAULT_WORKERS) + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise SystemExit( + f"stack {entry.get('id')!r}: `workers` must be a whole number >= 1, not {value!r}" + ) + return value + + def build_matrix(stacks: list[dict[str, Any]]) -> list[dict[str, Any]]: """Expand manifest entries into {stack x config x version} jobs.""" include = [] @@ -62,6 +76,7 @@ def build_matrix(stacks: list[dict[str, Any]]) -> list[dict[str, Any]]: "version": version, "markers": entry["markers"], "exec_in_container": entry.get("exec_in_container", False), + "workers": workers(entry), } ) return include diff --git a/tests/integration/stacks/_images/htcondor-ce/Dockerfile b/tests/integration/stacks/_images/htcondor-ce/Dockerfile new file mode 100644 index 0000000..2110478 --- /dev/null +++ b/tests/integration/stacks/_images/htcondor-ce/Dockerfile @@ -0,0 +1,52 @@ +# An HTCondor-CE in front of a one-machine HTCondor pool, for the `htcondor` stack. +# Rules: docs/dev/reference/integration-images.md +# +# The CE is the grid gateway interCEde submits to; its job router hands each job to the local +# pool, which runs it. Both run in this one container, started by entrypoint.sh (no systemd). +FROM almalinux:9 + +# Shared by every version, so above the ARGs: an ARG is part of the cache key of each later RUN. +# dnf leaves a mirror slower than minrate after `timeout` seconds, instead of finishing on it. +RUN printf '%s\n' 'fastestmirror=True' 'max_parallel_downloads=10' 'minrate=1M' 'timeout=15' \ + >> /etc/dnf/dnf.conf +# The metadata stays cached on purpose: the install below reuses it instead of downloading it again. +RUN set -eux; \ + dnf install -y epel-release dnf-plugins-core; \ + dnf config-manager --set-enabled crb + +# Leading edge, updated by Renovate (the `htcondor` group in renovate.json). +# renovate: datasource=github-tags depName=htcondor/htcondor +ARG HTCONDOR_VERSION=v25.14.1 +# renovate: datasource=github-tags depName=htcondor/htcondor-ce +ARG HTCONDOR_CE_VERSION=v25.7.0 + +# Long-term-support anchor, updated by hand: no `# renovate:` comment. +ARG HTCONDOR_LTS_VERSION=25.0.14 +ARG HTCONDOR_CE_LTS_VERSION=25.0.1 + +# Which of the two to build: `latest` or `lts`, from the stack's `versions` (compose passes it). +ARG BACKEND_VERSION=latest + +RUN set -eux; \ + case "$BACKEND_VERSION" in \ + latest) condor="${HTCONDOR_VERSION#v}"; ce="${HTCONDOR_CE_VERSION#v}"; series="${condor%%.*}.x" ;; \ + lts) condor="${HTCONDOR_LTS_VERSION}"; ce="${HTCONDOR_CE_LTS_VERSION}"; series="${condor%%.*}.0" ;; \ + *) echo "BACKEND_VERSION must be latest or lts, not '$BACKEND_VERSION'" >&2; exit 1 ;; \ + esac; \ + dnf install -y "https://research.cs.wisc.edu/htcondor/repo/${series}/htcondor-release-current.el9.noarch.rpm"; \ + dnf install -y "condor-${condor}" "htcondor-ce-${ce}" "htcondor-ce-condor-${ce}"; \ + dnf clean all; \ + rm -rf /var/cache/dnf + +# Jobs run as this unprivileged user: HTCondor refuses to run jobs as root. A token for +# submituser@users.htcondor.org (the CE's trust domain) maps to it. +RUN useradd --create-home submituser + +COPY pool.conf /etc/condor/config.d/90-intercede-pool.conf +COPY ce-router.conf /etc/condor-ce/config.d/90-intercede-router.conf +COPY entrypoint.sh /usr/local/bin/entrypoint.sh + +# The CE's single port (shared by all its daemons). +EXPOSE 9619 + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/tests/integration/stacks/_images/htcondor-ce/ce-router.conf b/tests/integration/stacks/_images/htcondor-ce/ce-router.conf new file mode 100644 index 0000000..6455af5 --- /dev/null +++ b/tests/integration/stacks/_images/htcondor-ce/ce-router.conf @@ -0,0 +1,6 @@ +# The pool behind the CE, in the same container (pool.conf). The packaged default names the +# schedd after the host, which the router cannot find (see SCHEDD_NAME in pool.conf). +JOB_ROUTER_SCHEDD2_NAME = pool@$(FULL_HOSTNAME) + +# Route new jobs every few seconds instead of every ten, so a test job reaches the pool quickly. +JOB_ROUTER_POLLING_PERIOD = 2 diff --git a/tests/integration/stacks/_images/htcondor-ce/entrypoint.sh b/tests/integration/stacks/_images/htcondor-ce/entrypoint.sh new file mode 100755 index 0000000..3aca6f9 --- /dev/null +++ b/tests/integration/stacks/_images/htcondor-ce/entrypoint.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Start the HTCondor pool and the HTCondor-CE in the foreground (no systemd), after writing a +# fresh client token to the shared credentials folder. Exits as soon as either daemon tree stops, +# so `docker compose` and the healthcheck see the failure. +set -euo pipefail + +CREDENTIALS="${CREDENTIALS_DIR:-/shared/credentials/htcondor}" +IDENTITY="${CLIENT_IDENTITY:-submituser@users.htcondor.org}" + +# The CE has no condor_ce_* wrapper for these two tools: run them with the CE's configuration. +ce() { CONDOR_CONFIG=/etc/condor-ce/condor_config "$@"; } + +# The CE signs ID tokens with its pool key. Create a new one at every start: credentials are +# throwaway and never part of the image (integration-images.md, rule 7). +install -d -m 0700 -o root -g root /etc/condor-ce/passwords.d +rm -f /etc/condor-ce/passwords.d/POOL +openssl rand -base64 48 | tr -d '\n' | ce condor_store_cred add -c -i - >/dev/null + +# The client's token, readable by everyone: the harness on the host copies it to a private file +# itself (integration-stacks.md, rule 6). +install -d -m 0755 "$CREDENTIALS" +ce condor_token_create -identity "$IDENTITY" > "$CREDENTIALS/token" +chmod 0644 "$CREDENTIALS/token" + +condor_master -f & # the pool +pool_pid=$! +/usr/share/condor-ce/condor_ce_startup -f & # the CE (checks its config, then runs its master) +ce_pid=$! + +# Daemon logs to stdout, so `docker compose logs` holds them all (integration-stacks.md, rule 4). +# HTCondor daemons can only log to files, and cannot open /dev/stdout as that file: they switch +# from root to the condor user, which may not reopen root's stdout. So the files are followed +# instead. `tail -F` waits for files that do not exist yet and prints a header per file. +follow() { + local tag="$1" dir="$2"; shift 2 + tail -n +1 -F "${@/#/$dir/}" 2>/dev/null | sed -u "s/^/[$tag] /" & +} +follow pool /var/log/condor MasterLog CollectorLog NegotiatorLog SchedLog StartLog \ + SharedPortLog ShadowLog StarterLog.slot1_1 StarterLog.slot1_2 StarterLog.slot1_3 StarterLog.slot1_4 +follow ce /var/log/condor-ce MasterLog CollectorLog SchedLog JobRouterLog SharedPortLog AuditLog + +trap 'kill "$pool_pid" "$ce_pid" 2>/dev/null; wait' TERM INT +wait -n "$pool_pid" "$ce_pid" +status=$? +echo "a daemon tree stopped (exit $status); stopping the container" >&2 +kill "$pool_pid" "$ce_pid" 2>/dev/null || true +exit "$status" diff --git a/tests/integration/stacks/_images/htcondor-ce/pool.conf b/tests/integration/stacks/_images/htcondor-ce/pool.conf new file mode 100644 index 0000000..805976d --- /dev/null +++ b/tests/integration/stacks/_images/htcondor-ce/pool.conf @@ -0,0 +1,13 @@ +# The batch pool behind the CE: one machine that is central manager, submit and execute node. +use ROLE: CentralManager +use ROLE: Submit +use ROLE: Execute +CONDOR_HOST = $(FULL_HOSTNAME) + +# The CE's job router finds this schedd by name (ce-router.conf). HTCondor resolves a plain +# name through DNS, and a container's reverse DNS gives another name than its hostname; a +# `name@host` name is taken as it is. +SCHEDD_NAME = pool@$(FULL_HOSTNAME) + +# Match jobs every few seconds instead of every minute, so a test job starts straight away. +NEGOTIATOR_INTERVAL = 2 diff --git a/tests/integration/stacks/htcondor/compose.yml b/tests/integration/stacks/htcondor/compose.yml new file mode 100644 index 0000000..f9bc43c --- /dev/null +++ b/tests/integration/stacks/htcondor/compose.yml @@ -0,0 +1,38 @@ +# The `htcondor` stack: an HTCondor-CE in front of a one-machine HTCondor pool. +# Rules: docs/dev/reference/integration-stacks.md +# The harness drives the service named `backend`: the CE, on 127.0.0.1:9619. +services: + backend: + # Until CI publishes images (#14): ghcr.io/diracgrid/intercede-testenv/htcondor-ce:. + # A local name per version, so `latest` and `lts` do not replace each other. + image: intercede-testenv/htcondor-ce:${BACKEND_VERSION:-latest} + build: + context: ../_images/htcondor-ce + args: + BACKEND_VERSION: ${BACKEND_VERSION:-latest} + hostname: backend + networks: + - stack + ports: + # Host-run harness: the CE advertises 127.0.0.1:9619 (config/basic/ce.conf), so the + # published port must be the same number. + - "127.0.0.1:9619:9619" + volumes: + # bind mount, not a named volume: the host-run harness must read the token + - ./credentials:/shared/credentials + # backend half of the configuration, next to its intercede-client.toml + - ./config/basic/ce.conf:/etc/condor-ce/config.d/99-basic.conf:ro + healthcheck: + # Ready when the CE's schedd is advertised and the pool has a slot to run jobs on. + test: + - CMD-SHELL + - >- + condor_ce_status -schedd -af Name | grep -q . && + condor_status -af Name | grep -q . + interval: 5s + timeout: 10s + retries: 24 + start_period: 30s + +networks: + stack: {} diff --git a/tests/integration/stacks/htcondor/config/basic/ce.conf b/tests/integration/stacks/htcondor/config/basic/ce.conf new file mode 100644 index 0000000..b565d12 --- /dev/null +++ b/tests/integration/stacks/htcondor/config/basic/ce.conf @@ -0,0 +1,19 @@ +# Backend half of the `basic` configuration: mounted into the CE as +# /etc/condor-ce/config.d/99-basic.conf. Changes together with intercede-client.toml +# (integration-stacks.md, rule 7). + +# Clients authenticate with an ID token (issue #23: the pure-IDTOKENS basic configuration). Out +# of the box the CE's collector accepts one, but its schedd takes only SCITOKENS and SSL from a +# client. FS stays for the CE's own daemons and tools inside the container. +SCHEDD.SEC_READ_AUTHENTICATION_METHODS = FS, TOKEN +SCHEDD.SEC_WRITE_AUTHENTICATION_METHODS = FS, TOKEN + +# The host reaches the CE through the published port 127.0.0.1:9619 (integration-stacks.md, +# rule 9). The CE's daemons all share that port, so advertising it works both from the host and +# from inside the container, where 127.0.0.1:9619 is the CE itself. +TCP_FORWARDING_HOST = 127.0.0.1 + +# The CE's schedd name, as clients pass it with `-name`. HTCondor tools look a plain name up in +# DNS and reject it when the host cannot resolve it (a container hostname, or `localhost`); a +# name of the form `name@host` is taken as it is. +SCHEDD_NAME = htcondor-ce@backend diff --git a/tests/integration/stacks/htcondor/config/basic/intercede-client.toml b/tests/integration/stacks/htcondor/config/basic/intercede-client.toml new file mode 100644 index 0000000..8026759 --- /dev/null +++ b/tests/integration/stacks/htcondor/config/basic/intercede-client.toml @@ -0,0 +1,22 @@ +# Client half of the `basic` configuration. See docs/dev/reference/integration-stacks.md +# Changes together with ce.conf, the backend half (rule 7). + +[stack] +id = "htcondor" # this stack's id in stacks.yml + +[backend] +kind = "htcondor-ce" +# The CE, published on the host (compose.yml) and advertised under this address (ce.conf). +endpoint = "127.0.0.1:9619" +# The CE's schedd name, as set in ce.conf. A `name@host` name: HTCondor tools reject a plain +# name the host cannot resolve, such as the container's hostname. +schedd = "htcondor-ce@backend" +# No `queue`: an HTCondor pool has no queues, and the CE routes every job to it. + +[credentials] +type = "token" # an ID token for submituser@users.htcondor.org (IDTOKENS) +path = "credentials/htcondor/" # holds `token`, minted at every container start + +[refusals] +# What this backend cannot honour; the shared suite checks it is refused at submit. +containers = true # no container_launcher configured above diff --git a/tests/integration/test_manifest.py b/tests/integration/test_manifest.py index 8304ae5..bda884f 100644 --- a/tests/integration/test_manifest.py +++ b/tests/integration/test_manifest.py @@ -2,7 +2,12 @@ from __future__ import annotations -from tests.integration.generate_matrix import STACKS_DIR, compose_env, load_manifest +from tests.integration.generate_matrix import ( + STACKS_DIR, + compose_env, + load_manifest, + workers, +) def test_entries_have_required_keys(): @@ -12,6 +17,11 @@ def test_entries_have_required_keys(): assert entry.get("markers"), f"stack {entry['id']}: `markers` is required" +def test_workers_are_valid(): + for entry in load_manifest(): + workers(entry) # raises SystemExit on a bad value + + def test_stack_ids_are_unique(): ids = [entry["id"] for entry in load_manifest()] assert len(ids) == len(set(ids)), f"duplicate stack ids in stacks.yml: {ids}"