Test: Integration stack for HTCondor

.github/workflows/integration.yml

@@ -10,6 +10,7 @@ on:
       - ".github/workflows/integration.yml"
       - "tests/conftest.py"
       - "pyproject.toml"
+      - "src/**"
 
 permissions:
   contents: read
@@ -66,10 +67,12 @@ jobs:
             # TODO(#13/#14): ship the suite in the image and exec as an unprivileged user.
             runner=(docker compose -f "$COMPOSE_FILE" exec -T backend pytest)
           else
-            runner=(pixi run -e py313 pytest)
+            # Independent tests, run side by side (stacks.yml `workers`).
+            runner=(pixi run -e py313 pytest -n "${{ matrix.workers }}")
           fi
           set +e
-          "${runner[@]}" tests/integration -m "${{ matrix.markers }}" \
+          # -rs --no-fold-skipped: name every skipped test and why.
+          "${runner[@]}" tests/integration -m "${{ matrix.markers }}" -rs --no-fold-skipped \
             --stack="${{ matrix.stack }}" --config="${{ matrix.config }}"
           rc=$?
           # Exit 5 = no tests selected; tolerated until the contract suite lands.

docs/dev/how-to/add-an-integration-stack.md

@@ -24,7 +24,7 @@ You do not need to change any CI workflow. CI finds the stack in `stacks.yml`.
 
    ```sh
    docker compose -f tests/integration/stacks/<id>/compose.yml up --wait
-   pixi run -e py313 pytest tests/integration -m "<markers>" --stack=<id> --config=basic
+   pixi run -e py313 pytest tests/integration -n 4 -m "<markers>" --stack=<id> --config=basic
    pixi run -e py313 pytest tests/integration/test_manifest.py
    ```
 

docs/dev/how-to/run-integration-tests-locally.md

@@ -16,9 +16,12 @@ CI runs the same commands as below, so you can reproduce any CI job on your mach
 2. Run the tests. Copy `<markers>` from the stack's entry in `stacks.yml`:
 
    ```sh
-   pixi run -e py313 pytest tests/integration -m "<markers>" --stack=<id> --config=basic
+   pixi run -e py313 pytest tests/integration -n 4 -m "<markers>" --stack=<id> --config=basic
    ```
 
+`-n 4` runs four tests at the same time, as CI does. Use the stack's `workers` value from
+`stacks.yml` if it sets one, or leave `-n` out to run one test at a time while debugging.
+
 For stacks with `exec_in_container: true` (the `local-*` stacks), run the tests inside the
 container instead:
 

docs/dev/reference/integration-stacks.md

@@ -44,6 +44,7 @@ tests/integration/
 | `versions` | no | Backend versions to test, for example `[lts, latest]`. Default: `[latest]`. |
 | `markers` | yes | `pytest -m` expression that selects the tests for this stack. |
 | `exec_in_container` | no | `true`: run pytest inside the backend container. Default: `false`. |
+| `workers` | no | How many tests run at the same time (pytest-xdist). Default: `4`. Set `1` for a backend that cannot take several jobs at once. |
 
 ## When a stack gets CI jobs
 

tests/integration/README.md

@@ -5,7 +5,7 @@ called a *stack*.
 
 ```sh
 docker compose -f tests/integration/stacks/<id>/compose.yml up --wait
-pixi run -e py313 pytest tests/integration -m "<markers>" --stack=<id> --config=basic
+pixi run -e py313 pytest tests/integration -n 4 -m "<markers>" --stack=<id> --config=basic
 ```
 
 Documentation:

tests/integration/generate_matrix.py

@@ -19,6 +19,10 @@
 STACKS_DIR = Path(__file__).parent / "stacks"
 
 REQUIRED_KEYS = ("id", "configs", "markers")
+# pytest-xdist workers per job. The tests are independent and every backend
+# queues concurrent jobs, so they run side by side; a stack whose backend
+# cannot take concurrent jobs sets `workers: 1`.
+DEFAULT_WORKERS = 4
 
 
 def load_manifest() -> list[dict[str, Any]]:
@@ -42,6 +46,16 @@ def is_ready(entry: dict[str, Any]) -> bool:
     ).is_dir()
 
 
+def workers(entry: dict[str, Any]) -> int:
+    """Return how many pytest-xdist workers the stack's tests run with."""
+    value = entry.get("workers", DEFAULT_WORKERS)
+    if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+        raise SystemExit(
+            f"stack {entry.get('id')!r}: `workers` must be a whole number >= 1, not {value!r}"
+        )
+    return value
+
+
 def build_matrix(stacks: list[dict[str, Any]]) -> list[dict[str, Any]]:
     """Expand manifest entries into {stack x config x version} jobs."""
     include = []
@@ -62,6 +76,7 @@ def build_matrix(stacks: list[dict[str, Any]]) -> list[dict[str, Any]]:
                         "version": version,
                         "markers": entry["markers"],
                         "exec_in_container": entry.get("exec_in_container", False),
+                        "workers": workers(entry),
                     }
                 )
     return include

tests/integration/stacks/_images/htcondor-ce/Dockerfile

@@ -0,0 +1,52 @@
+# An HTCondor-CE in front of a one-machine HTCondor pool, for the `htcondor` stack.
+# Rules: docs/dev/reference/integration-images.md
+#
+# The CE is the grid gateway interCEde submits to; its job router hands each job to the local
+# pool, which runs it. Both run in this one container, started by entrypoint.sh (no systemd).
+FROM almalinux:9
+
+# Shared by every version, so above the ARGs: an ARG is part of the cache key of each later RUN.
+# dnf leaves a mirror slower than minrate after `timeout` seconds, instead of finishing on it.
+RUN printf '%s\n' 'fastestmirror=True' 'max_parallel_downloads=10' 'minrate=1M' 'timeout=15' \
+      >> /etc/dnf/dnf.conf
+# The metadata stays cached on purpose: the install below reuses it instead of downloading it again.
+RUN set -eux; \
+    dnf install -y epel-release dnf-plugins-core; \
+    dnf config-manager --set-enabled crb
+
+# Leading edge, updated by Renovate (the `htcondor` group in renovate.json).
+# renovate: datasource=github-tags depName=htcondor/htcondor
+ARG HTCONDOR_VERSION=v25.14.1
+# renovate: datasource=github-tags depName=htcondor/htcondor-ce
+ARG HTCONDOR_CE_VERSION=v25.7.0
+
+# Long-term-support anchor, updated by hand: no `# renovate:` comment.
+ARG HTCONDOR_LTS_VERSION=25.0.14
+ARG HTCONDOR_CE_LTS_VERSION=25.0.1
+
+# Which of the two to build: `latest` or `lts`, from the stack's `versions` (compose passes it).
+ARG BACKEND_VERSION=latest
+
+RUN set -eux; \
+    case "$BACKEND_VERSION" in \
+        latest) condor="${HTCONDOR_VERSION#v}"; ce="${HTCONDOR_CE_VERSION#v}"; series="${condor%%.*}.x" ;; \
+        lts)    condor="${HTCONDOR_LTS_VERSION}"; ce="${HTCONDOR_CE_LTS_VERSION}"; series="${condor%%.*}.0" ;; \
+        *)      echo "BACKEND_VERSION must be latest or lts, not '$BACKEND_VERSION'" >&2; exit 1 ;; \
+    esac; \
+    dnf install -y "https://research.cs.wisc.edu/htcondor/repo/${series}/htcondor-release-current.el9.noarch.rpm"; \
+    dnf install -y "condor-${condor}" "htcondor-ce-${ce}" "htcondor-ce-condor-${ce}"; \
+    dnf clean all; \
+    rm -rf /var/cache/dnf
+
+# Jobs run as this unprivileged user: HTCondor refuses to run jobs as root. A token for
+# submituser@users.htcondor.org (the CE's trust domain) maps to it.
+RUN useradd --create-home submituser
+
+COPY pool.conf /etc/condor/config.d/90-intercede-pool.conf
+COPY ce-router.conf /etc/condor-ce/config.d/90-intercede-router.conf
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+
+# The CE's single port (shared by all its daemons).
+EXPOSE 9619
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

tests/integration/stacks/_images/htcondor-ce/ce-router.conf

@@ -0,0 +1,6 @@
+# The pool behind the CE, in the same container (pool.conf). The packaged default names the
+# schedd after the host, which the router cannot find (see SCHEDD_NAME in pool.conf).
+JOB_ROUTER_SCHEDD2_NAME = pool@$(FULL_HOSTNAME)
+
+# Route new jobs every few seconds instead of every ten, so a test job reaches the pool quickly.
+JOB_ROUTER_POLLING_PERIOD = 2

tests/integration/stacks/_images/htcondor-ce/entrypoint.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Start the HTCondor pool and the HTCondor-CE in the foreground (no systemd), after writing a
+# fresh client token to the shared credentials folder. Exits as soon as either daemon tree stops,
+# so `docker compose` and the healthcheck see the failure.
+set -euo pipefail
+
+CREDENTIALS="${CREDENTIALS_DIR:-/shared/credentials/htcondor}"
+IDENTITY="${CLIENT_IDENTITY:-submituser@users.htcondor.org}"
+
+# The CE has no condor_ce_* wrapper for these two tools: run them with the CE's configuration.
+ce() { CONDOR_CONFIG=/etc/condor-ce/condor_config "$@"; }
+
+# The CE signs ID tokens with its pool key. Create a new one at every start: credentials are
+# throwaway and never part of the image (integration-images.md, rule 7).
+install -d -m 0700 -o root -g root /etc/condor-ce/passwords.d
+rm -f /etc/condor-ce/passwords.d/POOL
+openssl rand -base64 48 | tr -d '\n' | ce condor_store_cred add -c -i - >/dev/null
+
+# The client's token, readable by everyone: the harness on the host copies it to a private file
+# itself (integration-stacks.md, rule 6).
+install -d -m 0755 "$CREDENTIALS"
+ce condor_token_create -identity "$IDENTITY" > "$CREDENTIALS/token"
+chmod 0644 "$CREDENTIALS/token"
+
+condor_master -f &                          # the pool
+pool_pid=$!
+/usr/share/condor-ce/condor_ce_startup -f & # the CE (checks its config, then runs its master)
+ce_pid=$!
+
+# Daemon logs to stdout, so `docker compose logs` holds them all (integration-stacks.md, rule 4).
+# HTCondor daemons can only log to files, and cannot open /dev/stdout as that file: they switch
+# from root to the condor user, which may not reopen root's stdout. So the files are followed
+# instead. `tail -F` waits for files that do not exist yet and prints a header per file.
+follow() {
+    local tag="$1" dir="$2"; shift 2
+    tail -n +1 -F "${@/#/$dir/}" 2>/dev/null | sed -u "s/^/[$tag] /" &
+}
+follow pool /var/log/condor MasterLog CollectorLog NegotiatorLog SchedLog StartLog \
+    SharedPortLog ShadowLog StarterLog.slot1_1 StarterLog.slot1_2 StarterLog.slot1_3 StarterLog.slot1_4
+follow ce /var/log/condor-ce MasterLog CollectorLog SchedLog JobRouterLog SharedPortLog AuditLog
+
+trap 'kill "$pool_pid" "$ce_pid" 2>/dev/null; wait' TERM INT
+wait -n "$pool_pid" "$ce_pid"
+status=$?
+echo "a daemon tree stopped (exit $status); stopping the container" >&2
+kill "$pool_pid" "$ce_pid" 2>/dev/null || true
+exit "$status"

tests/integration/stacks/_images/htcondor-ce/pool.conf

@@ -0,0 +1,13 @@
+# The batch pool behind the CE: one machine that is central manager, submit and execute node.
+use ROLE: CentralManager
+use ROLE: Submit
+use ROLE: Execute
+CONDOR_HOST = $(FULL_HOSTNAME)
+
+# The CE's job router finds this schedd by name (ce-router.conf). HTCondor resolves a plain
+# name through DNS, and a container's reverse DNS gives another name than its hostname; a
+# `name@host` name is taken as it is.
+SCHEDD_NAME = pool@$(FULL_HOSTNAME)
+
+# Match jobs every few seconds instead of every minute, so a test job starts straight away.
+NEGOTIATOR_INTERVAL = 2

tests/integration/stacks/htcondor/compose.yml

@@ -0,0 +1,38 @@
+# The `htcondor` stack: an HTCondor-CE in front of a one-machine HTCondor pool.
+# Rules: docs/dev/reference/integration-stacks.md
+# The harness drives the service named `backend`: the CE, on 127.0.0.1:9619.
+services:
+  backend:
+    # Until CI publishes images (#14): ghcr.io/diracgrid/intercede-testenv/htcondor-ce:<version>.
+    # A local name per version, so `latest` and `lts` do not replace each other.
+    image: intercede-testenv/htcondor-ce:${BACKEND_VERSION:-latest}
+    build:
+      context: ../_images/htcondor-ce
+      args:
+        BACKEND_VERSION: ${BACKEND_VERSION:-latest}
+    hostname: backend
+    networks:
+      - stack
+    ports:
+      # Host-run harness: the CE advertises 127.0.0.1:9619 (config/basic/ce.conf), so the
+      # published port must be the same number.
+      - "127.0.0.1:9619:9619"
+    volumes:
+      # bind mount, not a named volume: the host-run harness must read the token
+      - ./credentials:/shared/credentials
+      # backend half of the configuration, next to its intercede-client.toml
+      - ./config/basic/ce.conf:/etc/condor-ce/config.d/99-basic.conf:ro
+    healthcheck:
+      # Ready when the CE's schedd is advertised and the pool has a slot to run jobs on.
+      test:
+        - CMD-SHELL
+        - >-
+          condor_ce_status -schedd -af Name | grep -q . &&
+          condor_status -af Name | grep -q .
+      interval: 5s
+      timeout: 10s
+      retries: 24
+      start_period: 30s
+
+networks:
+  stack: {}

tests/integration/stacks/htcondor/config/basic/ce.conf

@@ -0,0 +1,19 @@
+# Backend half of the `basic` configuration: mounted into the CE as
+# /etc/condor-ce/config.d/99-basic.conf. Changes together with intercede-client.toml
+# (integration-stacks.md, rule 7).
+
+# Clients authenticate with an ID token (issue #23: the pure-IDTOKENS basic configuration). Out
+# of the box the CE's collector accepts one, but its schedd takes only SCITOKENS and SSL from a
+# client. FS stays for the CE's own daemons and tools inside the container.
+SCHEDD.SEC_READ_AUTHENTICATION_METHODS = FS, TOKEN
+SCHEDD.SEC_WRITE_AUTHENTICATION_METHODS = FS, TOKEN
+
+# The host reaches the CE through the published port 127.0.0.1:9619 (integration-stacks.md,
+# rule 9). The CE's daemons all share that port, so advertising it works both from the host and
+# from inside the container, where 127.0.0.1:9619 is the CE itself.
+TCP_FORWARDING_HOST = 127.0.0.1
+
+# The CE's schedd name, as clients pass it with `-name`. HTCondor tools look a plain name up in
+# DNS and reject it when the host cannot resolve it (a container hostname, or `localhost`); a
+# name of the form `name@host` is taken as it is.
+SCHEDD_NAME = htcondor-ce@backend

tests/integration/stacks/htcondor/config/basic/intercede-client.toml

@@ -0,0 +1,22 @@
+# Client half of the `basic` configuration. See docs/dev/reference/integration-stacks.md
+# Changes together with ce.conf, the backend half (rule 7).
+
+[stack]
+id = "htcondor" # this stack's id in stacks.yml
+
+[backend]
+kind = "htcondor-ce"
+# The CE, published on the host (compose.yml) and advertised under this address (ce.conf).
+endpoint = "127.0.0.1:9619"
+# The CE's schedd name, as set in ce.conf. A `name@host` name: HTCondor tools reject a plain
+# name the host cannot resolve, such as the container's hostname.
+schedd = "htcondor-ce@backend"
+# No `queue`: an HTCondor pool has no queues, and the CE routes every job to it.
+
+[credentials]
+type = "token"                  # an ID token for submituser@users.htcondor.org (IDTOKENS)
+path = "credentials/htcondor/"  # holds `token`, minted at every container start
+
+[refusals]
+# What this backend cannot honour; the shared suite checks it is refused at submit.
+containers = true # no container_launcher configured above

tests/integration/test_manifest.py

@@ -2,7 +2,12 @@
 
 from __future__ import annotations
 
-from tests.integration.generate_matrix import STACKS_DIR, compose_env, load_manifest
+from tests.integration.generate_matrix import (
+    STACKS_DIR,
+    compose_env,
+    load_manifest,
+    workers,
+)
 
 
 def test_entries_have_required_keys():
@@ -12,6 +17,11 @@ def test_entries_have_required_keys():
         assert entry.get("markers"), f"stack {entry['id']}: `markers` is required"
 
 
+def test_workers_are_valid():
+    for entry in load_manifest():
+        workers(entry)  # raises SystemExit on a bad value
+
+
 def test_stack_ids_are_unique():
     ids = [entry["id"] for entry in load_manifest()]
     assert len(ids) == len(set(ids)), f"duplicate stack ids in stacks.yml: {ids}"