diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c4b32cc..d9d8485 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,6 +10,7 @@ on: - ".github/workflows/integration.yml" - "tests/conftest.py" - "pyproject.toml" + - "src/**" permissions: contents: read @@ -66,10 +67,12 @@ jobs: # TODO(#13/#14): ship the suite in the image and exec as an unprivileged user. runner=(docker compose -f "$COMPOSE_FILE" exec -T backend pytest) else - runner=(pixi run -e py313 pytest) + # Independent tests, run side by side (stacks.yml `workers`). + runner=(pixi run -e py313 pytest -n "${{ matrix.workers }}") fi set +e - "${runner[@]}" tests/integration -m "${{ matrix.markers }}" \ + # -rs --no-fold-skipped: name every skipped test and why. + "${runner[@]}" tests/integration -m "${{ matrix.markers }}" -rs --no-fold-skipped \ --stack="${{ matrix.stack }}" --config="${{ matrix.config }}" rc=$? # Exit 5 = no tests selected; tolerated until the contract suite lands. diff --git a/docs/dev/explanations/integration-testing.md b/docs/dev/explanations/integration-testing.md index 575093d..6cad600 100644 --- a/docs/dev/explanations/integration-testing.md +++ b/docs/dev/explanations/integration-testing.md @@ -21,14 +21,17 @@ know which version caused the problem. Some backends do not support every feature. The tests check for a feature in the same way that users of the library do (`isinstance` against a `runtime_checkable` protocol), and skip the test -when the backend does not have it. The helper for this will be added in phase 2, together with the -first tests that use it. +when the backend does not have it. A test asks for a capability by taking its fixture +(`output_retriever`, `killable`, `purgeable`). ## Phases -- **Phase 0 (now).** The interCEde protocols do not exist yet. Each stack is tested with the - backend's own command-line tools (`arcsub`, `arcstat`, `arcget`, `condor_submit`, `sbatch`). -- **Phase 2.** One set of tests, written against the interCEde protocols, runs on every stack - (`test_submit.py`, `test_status.py`, `test_output.py`, `test_kill.py`). It replaces the phase 0 - tests. Behaviour that only one backend has (for example HTCondor's one-time output download) - gets its own test and marker (`destructive_fetch`), not an `if` inside a shared test. +- **Phase 0.** Before the interCEde protocols existed, a stack was tested with the backend's own + command-line tools (`arcsub`, `arcstat`, `arcget`, `condor_submit`, `sbatch`). +- **Phase 2 (now).** One set of tests, written against the interCEde protocols, runs on every stack + (`test_submit.py`, `test_status.py`, `test_output.py`, `test_kill.py`, `test_purge.py`, and + `test_errors.py` for what a consumer sees when things go wrong). These tests carry the + `contract` marker, which selects them whatever a stack's `markers` say. A new backend writes no + tests of its own: its stack runs this suite. Behaviour that only one backend has (for example + HTCondor's one-time output download) gets its own test and marker (`destructive_fetch`), not an + `if` inside a shared test. diff --git a/docs/dev/how-to/add-an-integration-stack.md b/docs/dev/how-to/add-an-integration-stack.md index ff9b8e4..29fed21 100644 --- a/docs/dev/how-to/add-an-integration-stack.md +++ b/docs/dev/how-to/add-an-integration-stack.md @@ -24,7 +24,7 @@ You do not need to change any CI workflow. CI finds the stack in `stacks.yml`. ```sh docker compose -f tests/integration/stacks//compose.yml up --wait - pixi run -e py313 pytest tests/integration -m "" --stack= --config=basic + pixi run -e py313 pytest tests/integration -n 4 -m "" --stack= --config=basic pixi run -e py313 pytest tests/integration/test_manifest.py ``` @@ -35,8 +35,9 @@ You do not need to change any CI workflow. CI finds the stack in `stacks.yml`. - [ ] The images use Docker, not Podman. - [ ] `intercede-client.toml` is filled in: `id`, endpoint, queue, credential type and path. - [ ] `pytest tests/integration/test_manifest.py` passes. -- [ ] At least one test runs for this stack (phase 0: a test that uses the backend's own - command-line tools). Without one, the CI job passes but tests nothing. +- [ ] The shared suite passes on this stack (`pytest tests/integration -m "" --stack=`). + You do not write tests for a new backend: the stack runs the `contract` tests, skipping + those for capabilities the backend does not have. - [ ] If your pull request had its own workflow for this stack, remove it. `integration.yml` runs on every pull request that changes `tests/integration/`. It does not block merging yet (#14), so keep your own workflow only if you need a check that blocks merging. diff --git a/docs/dev/how-to/run-integration-tests-locally.md b/docs/dev/how-to/run-integration-tests-locally.md index 6b720b4..fe00769 100644 --- a/docs/dev/how-to/run-integration-tests-locally.md +++ b/docs/dev/how-to/run-integration-tests-locally.md @@ -16,9 +16,12 @@ CI runs the same commands as below, so you can reproduce any CI job on your mach 2. Run the tests. Copy `` from the stack's entry in `stacks.yml`: ```sh - pixi run -e py313 pytest tests/integration -m "" --stack= --config=basic + pixi run -e py313 pytest tests/integration -n 4 -m "" --stack= --config=basic ``` +`-n 4` runs four tests at the same time, as CI does. Use the stack's `workers` value from +`stacks.yml` if it sets one, or leave `-n` out to run one test at a time while debugging. + For stacks with `exec_in_container: true` (the `local-*` stacks), run the tests inside the container instead: diff --git a/docs/dev/reference/integration-stacks.md b/docs/dev/reference/integration-stacks.md index fa8fb84..311d00d 100644 --- a/docs/dev/reference/integration-stacks.md +++ b/docs/dev/reference/integration-stacks.md @@ -44,6 +44,7 @@ tests/integration/ | `versions` | no | Backend versions to test, for example `[lts, latest]`. Default: `[latest]`. | | `markers` | yes | `pytest -m` expression that selects the tests for this stack. | | `exec_in_container` | no | `true`: run pytest inside the backend container. Default: `false`. | +| `workers` | no | How many tests run at the same time (pytest-xdist). Default: `4`. Set `1` for a backend that cannot take several jobs at once. | ## When a stack gets CI jobs @@ -112,9 +113,20 @@ Defined in `tests/conftest.py` (options) and `tests/integration/conftest.py` (fi | `stack` | fixture | The stack's entry from `stacks.yml`. | | `stack_dir` | fixture | Folder with the stack's `compose.yml`. | | `client_config` | fixture | The stack's `intercede-client.toml`, already parsed. | +| `refusals` | fixture | What `[refusals]` in `intercede-client.toml` says the backend cannot honour: `containers`, `url_inputs`, `url_outputs`. The shared suite checks each is refused at submit. | +| `credentials` | fixture | A credential provider holding what `[credentials]` points to (types: `token`). | +| `backend` | fixture | The backend `[backend]` describes, built by the registry and open. | +| `credential` | fixture | The credential `[credentials]` points to, read from the stack's credentials folder. | +| `rejected_credentials` | fixture | A provider holding a credential of the right kind that the resource rejects. | +| `unreachable_config` | fixture | `[backend]`, pointed at a closed port, as when the resource is down. | +| `output_retriever`, `killable`, `purgeable` | fixture | The backend as that capability; skips the test when the backend does not have it. | If `intercede-client.toml` is missing, the test fails in CI (when `$CI` is set) and is skipped locally. In CI, a job where every test is skipped would look green but test nothing. -Markers, defined in `pyproject.toml`: `remote`, `scheduler`, `arc`, `htcondor`, `slurm`, `ssh`, -`destructive_fetch`. +Markers, defined in `pyproject.toml`: `contract`, `remote`, `scheduler`, `arc`, `htcondor`, +`slurm`, `ssh`, `destructive_fetch`. + +Tests marked `contract` are the shared suite and run on every stack: with `--stack`, they get the +markers that the stack's `markers` expression asks for, so `-m ""` selects them. The +other markers select tests for behaviour only some backends have. diff --git a/docs/user/reference/api.md b/docs/user/reference/api.md index 824c73e..c04f65a 100644 --- a/docs/user/reference/api.md +++ b/docs/user/reference/api.md @@ -19,6 +19,14 @@ Anything else, including every name starting with `_`, is internal. ::: intercede.registry +## Credentials + +::: intercede.auth.models + +::: intercede.auth.protocols + +::: intercede.auth.providers + ## Exceptions ::: intercede.exceptions diff --git a/pyproject.toml b/pyproject.toml index a86f715..00490c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ markers = [ "backend_only: restrict a test to the named backend fixture(s)", "xfail_backend: strict xfail a test on the named backend fixture(s)", # integration suite (selected per stack in tests/integration/stacks.yml) + "contract: the shared integration suite, run on every stack whatever its markers", "remote: drives a remote CE backend (grid gateway: ARC, HTCondor-CE)", "scheduler: drives a Transport x Scheduler composition (BatchBackend)", "arc: requires an ARC CE", @@ -144,6 +145,11 @@ py313 = { features = ["testing", "py313"] } py314 = { features = ["testing", "py314"] } docs = { features = ["docs"], solve-group = "default" } +[tool.pixi.feature.testing.dependencies] +# The HTCondor tools (condor_submit, condor_q, ...) the htcondor-ce backend runs; +# the integration suite drives the backend against a real CE with them. +htcondor-utils = "*" + [tool.pixi.feature.testing.pypi-dependencies] intercede = { path = ".", editable = true } diff --git a/src/intercede/_backends/_htcondor/__init__.py b/src/intercede/_backends/_htcondor/__init__.py new file mode 100644 index 0000000..743a1fd --- /dev/null +++ b/src/intercede/_backends/_htcondor/__init__.py @@ -0,0 +1,11 @@ +"""What interCEde knows about HTCondor, shared by every backend that drives it. + +Plain functions and data only, with no dependency on any backend class: the +HTCondor-CE backend uses them directly, and the HTCondor scheduler of the +composed backend (issue #21) will use the same ones over its transport. They +share these functions, never a subtype. + +- ``submit``: a submit description built from a ``SubmissionSpec`` +- ``commands``: the argv of each HTCondor tool, and parsers for their output +- ``status``: the state mapping from job ads to ``JobStatus`` +""" diff --git a/src/intercede/_backends/_htcondor/commands.py b/src/intercede/_backends/_htcondor/commands.py new file mode 100644 index 0000000..8c76276 --- /dev/null +++ b/src/intercede/_backends/_htcondor/commands.py @@ -0,0 +1,117 @@ +"""The argv of each HTCondor tool, and parsers for what they print. + +``remote`` is how a tool reaches its schedd: ``schedd_options(pool, name)`` +for a remote schedd, as an HTCondor-CE is reached, or ``()`` for the schedd +of the host the tool runs on, as the composed backend reaches it. +""" + +from __future__ import annotations + +import re +from collections.abc import Sequence + +from intercede.models import JobID + +# Ids per command line, so a status poll of thousands of jobs stays bounded. +CHUNK = 100 + + +def schedd_options(pool: str, name: str) -> list[str]: + """Reach the schedd ``name`` through the collector ``pool`` (host:port).""" + return ["-pool", pool, "-name", name] + + +def submit( + description: str, pool: str | None = None, name: str | None = None +) -> list[str]: + """Submit a description file and spool its inputs to the schedd. + + Spooling copies the inputs at submit time, so nothing on the submitting + host is needed afterwards. With ``pool`` and ``name`` the schedd is a + remote one (``-remote`` implies spooling); without, it is the local one. + """ + if (pool is None) != (name is None): + raise ValueError("a remote schedd needs both pool and name") + + command = ["condor_submit", "-terse"] + + if pool is None or name is None: + command.append("-spool") + else: + command.extend(["-pool", pool, "-remote", name]) + + command.append(description) + return command + + +def query( + ids: Sequence[JobID], attributes: Sequence[str], remote: Sequence[str] +) -> list[str]: + """Read the ads of jobs still in the queue.""" + return ["condor_q", *remote, *ids, "-attributes", ",".join(attributes), "-json"] + + +def history( + ids: Sequence[JobID], attributes: Sequence[str], remote: Sequence[str] +) -> list[str]: + """Read the ads of jobs that have left the queue.""" + return [ + "condor_history", + *remote, + *ids, + "-attributes", + ",".join(attributes), + "-json", + ] + + +def remove(ids: Sequence[JobID], remote: Sequence[str]) -> list[str]: + """Remove jobs: cancels active ones, releases finished ones.""" + return ["condor_rm", *remote, *ids] + + +def transfer_data(ids: Sequence[JobID], remote: Sequence[str]) -> list[str]: + """Fetch the spooled output of jobs into the directory they were submitted from.""" + return ["condor_transfer_data", *remote, *ids] + + +def chunks(ids: Sequence[JobID]) -> list[Sequence[JobID]]: + """Split ids into groups of at most ``CHUNK``.""" + return [ids[start : start + CHUNK] for start in range(0, len(ids), CHUNK)] + + +_TERSE = re.compile( + r"\s*(?P\d+)\.(?P\d+)" + r"\s*-\s*" + r"(?P\d+)\.(?P\d+)\s*" +) + + +def parse_submit(raw: str) -> list[JobID]: + """Parse job IDs from condor_submit -terse output, e.g. '12.0 - 12.3'.""" + lines = [line for line in raw.splitlines() if line.strip()] + + if len(lines) != 1: + raise ValueError(f"unexpected condor_submit output: {raw.strip()!r}") + + match = _TERSE.fullmatch(lines[0]) + if match is None: + raise ValueError(f"unexpected condor_submit output: {raw.strip()!r}") + + cluster = int(match["cluster"]) + first = int(match["first"]) + last_cluster = int(match["last_cluster"]) + last = int(match["last"]) + + if cluster != last_cluster or last < first: + raise ValueError(f"unexpected condor_submit output: {raw.strip()!r}") + + return [JobID(f"{cluster}.{proc}") for proc in range(first, last + 1)] + + +_REMOVED = re.compile(r"^Job (\d+\.\d+) marked for removal", re.MULTILINE) + + +def parse_remove(raw: str) -> set[JobID]: + """Return the ids ``condor_rm`` reports as removed; any other id was not.""" + return {JobID(job) for job in _REMOVED.findall(raw)} diff --git a/src/intercede/_backends/_htcondor/status.py b/src/intercede/_backends/_htcondor/status.py new file mode 100644 index 0000000..40d8aca --- /dev/null +++ b/src/intercede/_backends/_htcondor/status.py @@ -0,0 +1,124 @@ +"""From HTCondor job ads to ``JobStatus``. + +Reading only: the state is taken from what HTCondor records about a job, never +from a policy interCEde sets on it. A finished job reads as FAILED when its +payload exited non-zero or was killed by a signal, from ``ExitCode`` and +``ExitBySignal``. + +Ported from DIRAC's ``BatchSystems/Condor.py`` (``STATES_MAP`` and +``getCondorStatus``), with differences: DIRAC puts a payload that fails on +hold to recognise it, where the exit code is read here instead; the states +DIRAC leaves unmapped (6, 7) are mapped; and a removed job reports what it did +before it was removed, so a purged job keeps its result. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from typing import Any + +from intercede.models import JobID, JobStatus + +# The ad attributes every status query asks for. +STATE_ATTRIBUTES: Sequence[str] = ( + "ClusterId", + "ProcId", + "JobStatus", + "LastJobStatus", + "ExitCode", + "ExitBySignal", + "HoldReasonCode", + "HoldReason", +) + +# HTCondor's JobStatus codes. +_IDLE, _RUNNING, _REMOVED, _COMPLETED, _HELD, _TRANSFERRING, _SUSPENDED = range(1, 8) + +_STATES: Mapping[int, JobStatus] = { + _IDLE: JobStatus.WAITING, + _RUNNING: JobStatus.RUNNING, + _TRANSFERRING: JobStatus.RUNNING, # transferring output after running + _SUSPENDED: JobStatus.WAITING, +} + +_HOLD_SPOOLING = 16 # HoldReasonCode: input files are still being spooled + + +def job_id(ad: Mapping[str, Any]) -> JobID: + """Return the ``cluster.proc`` id of a job ad.""" + return JobID(f"{ad['ClusterId']}.{ad['ProcId']}") + + +def job_status(ad: Mapping[str, Any]) -> JobStatus: + """Map one job ad to the common status.""" + code = ad.get("JobStatus") + if code == _COMPLETED: + return _result(ad) + if code == _REMOVED: + # purge() removes finished jobs to release them, after which the + # history only says "removed": the state it left tells the result. + return ( + _result(ad) if ad.get("LastJobStatus") == _COMPLETED else JobStatus.ABORTED + ) + if code == _HELD: + # Held by the site or by HTCondor, not by interCEde: the job is stuck, + # except while its inputs are still being spooled. + return ( + JobStatus.WAITING + if ad.get("HoldReasonCode") == _HOLD_SPOOLING + else JobStatus.ABORTED + ) + return ( + _STATES.get(code, JobStatus.UNKNOWN) + if isinstance(code, int) + else JobStatus.UNKNOWN + ) + + +def is_removed(ad: Mapping[str, Any]) -> bool: + """Tell whether a job was removed, even while it is still in the queue. + + ``condor_rm`` only marks a job for removal: the schedd drops it from the + queue a moment later. In between it is already released. + + Args: + ad: A job ad with ``JobStatus``. + + Returns: + True when the job has been removed. + + """ + return ad.get("JobStatus") == _REMOVED + + +def _result(ad: Mapping[str, Any]) -> JobStatus: + """Tell DONE from FAILED for a job whose payload has finished.""" + if ad.get("ExitBySignal") is True: + return JobStatus.FAILED + exit_code = ad.get("ExitCode") + return ( + JobStatus.FAILED + if isinstance(exit_code, int) and exit_code != 0 + else JobStatus.DONE + ) + + +def hold_reason(ad: Mapping[str, Any]) -> str | None: + """Return why a held job is held, if it is.""" + reason = ad.get("HoldReason") + return reason if ad.get("JobStatus") == _HELD and isinstance(reason, str) else None + + +def parse_ads(raw: str) -> list[dict[str, Any]]: + """Parse ``condor_q -json`` or ``condor_history -json`` output. + + ``condor_q`` prints nothing at all when no job matches, and + ``condor_history`` prints an empty list. + """ + if not raw.strip(): + return [] + ads = json.loads(raw) + if not isinstance(ads, list) or not all(isinstance(ad, dict) for ad in ads): + raise ValueError("expected a JSON list of job ads") + return ads diff --git a/src/intercede/_backends/_htcondor/submit.py b/src/intercede/_backends/_htcondor/submit.py new file mode 100644 index 0000000..0fa5a0f --- /dev/null +++ b/src/intercede/_backends/_htcondor/submit.py @@ -0,0 +1,219 @@ +"""Submit descriptions: a ``SubmissionSpec`` written in HTCondor's submit language. + +One description submits every copy of a spec as one cluster, one process per +copy. What differs between copies (environment, secrets, working directory) is +expanded per process through macros indexed by ``$(ProcId)``, so the copies +stay one bulk submission. + +A description only says what to run and where its files go. It sets no +policy on the job's lifetime: no holds, no automatic removal, no retries. +Those are the consumer's decisions. The one exception is that a finished job +stays in the queue until it is purged, so its output can be collected again. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field + + +def quote_arguments(arguments: Sequence[str]) -> str: + """Quote command arguments using HTCondor's new syntax.""" + quoted_arguments = [_quote_item(argument) for argument in arguments] + + joined_arguments = " ".join(quoted_arguments) + return f'"{joined_arguments}"' + + +def quote_environment(environment: Mapping[str, str]) -> str: + """Quote environment variables using HTCondor's new syntax.""" + for name in environment: + _validate_environment_name(name) + + quoted_variables = [ + f"{name}={_quote_item(value)}" for name, value in sorted(environment.items()) + ] + + joined_variables = " ".join(quoted_variables) + return f'"{joined_variables}"' + + +def _validate_environment_name(name: str) -> None: + """Raise ValueError if the environment variable name is invalid.""" + is_valid = bool(name) and "=" not in name and name.replace("_", "").isalnum() + + if not is_valid: + raise ValueError(f"{name!r} is not a valid environment variable name") + + +def _quote_item(value: str) -> str: + """Escape and quote a single argument or environment variable value.""" + _refuse_newlines(value) + + escaped_value = value.replace('"', '""') + escaped_value = escaped_value.replace("'", "''") + escaped_value = _escape_macros(escaped_value) + + return f"'{escaped_value}'" + + +def _escape_macros(value: str) -> str: + # "$(...)" is macro syntax in a submit description; $(DOLLAR) is a literal "$". + return value.replace("$", "$(DOLLAR)") + + +def _refuse_newlines(value: str) -> None: + if "\n" in value or "\r" in value: + raise ValueError("a submit description value cannot contain a line break") + + +@dataclass(frozen=True) +class CopyPlan: + """What one process of the cluster gets on its own.""" + + initialdir: str # absolute; outputs come back here (SUBMIT_Iwd) + environment: Mapping[str, str] = field(default_factory=dict) + # Inputs of this copy only, relative to ``initialdir`` (e.g. "secrets/"). + inputs: Sequence[str] = () + + +@dataclass(frozen=True) +class SubmitPlan: + """Everything a submit description says, resolved to HTCondor's terms.""" + + executable: str + transfer_executable: bool + arguments: Sequence[str] + copies: Sequence[CopyPlan] + inputs: Sequence[str] = () # shared by every copy: absolute paths or URLs + stdout: str = "$(ClusterId).$(ProcId).out" + stderr: str = "$(ClusterId).$(ProcId).err" + log: str = "$(ClusterId).$(ProcId).log" + # None: every file the payload creates comes back; empty: none at all. + outputs: Sequence[str] | None = () + output_remaps: Mapping[str, str] = field(default_factory=dict) # name -> URL + cpus: int = 1 + memory_mb: int | None = None + wall_time_s: int | None = None + queue: str | None = None + + +def render(plan: SubmitPlan) -> str: + """Write ``plan`` as a submit description that queues one process per copy.""" + _validate_plan(plan) + + lines = _render_basics(plan) + lines.extend(_render_copies(plan)) + lines.extend(_render_file_transfer(plan)) + lines.extend(_render_resources(plan)) + lines.extend(_render_queue(plan)) + + return "\n".join(lines) + "\n" + + +def _join(items: Sequence[str]) -> str: + for item in items: + _refuse_newlines(item) + if "," in item: + raise ValueError(f"{item!r}: HTCondor file lists cannot hold a comma") + return ", ".join(_escape_macros(item) for item in items) + + +def _bool(value: bool) -> str: + return "true" if value else "false" + + +def _classad_string(value: str) -> str: + _refuse_newlines(value) + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return '"' + _escape_macros(escaped) + '"' + + +def _validate_plan(plan: SubmitPlan) -> None: + if not plan.copies: + raise ValueError("A submit description needs at least one copy") + + +def _render_basics(plan: SubmitPlan) -> list[str]: + lines = [ + "universe = vanilla", + f"executable = {_escape_macros(plan.executable)}", + f"transfer_executable = {_bool(plan.transfer_executable)}", + ] + + if plan.arguments: + lines.append(f"arguments = {quote_arguments(plan.arguments)}") + + return lines + + +def _render_copies(plan: SubmitPlan) -> list[str]: + lines = [] + + for index, copy in enumerate(plan.copies): + _refuse_newlines(copy.initialdir) + lines.extend( + [ + f"ic_dir_{index} = {_escape_macros(copy.initialdir)}", + f"ic_env_{index} = {quote_environment(copy.environment)}", + f"ic_in_{index} = {_join(copy.inputs)}", + ] + ) + + lines.extend( + ["initialdir = $(ic_dir_$(ProcId))", "environment = $(ic_env_$(ProcId))"] + ) + + return lines + + +def _render_file_transfer(plan: SubmitPlan) -> list[str]: + shared_inputs = _join(plan.inputs) + copy_inputs = "$(ic_in_$(ProcId))" + inputs = f"{shared_inputs}, {copy_inputs}" if shared_inputs else copy_inputs + + lines = [ + "should_transfer_files = YES", + "when_to_transfer_output = ON_EXIT_OR_EVICT", + f"transfer_input_files = {inputs}", + f"output = {plan.stdout}", + f"error = {plan.stderr}", + f"log = {plan.log}", + ] + + if plan.outputs is not None: + output_files = _join(plan.outputs) or '""' + lines.append(f"transfer_output_files = {output_files}") + + if plan.output_remaps: + remaps = "; ".join( + f"{name} = {url}" for name, url in sorted(plan.output_remaps.items()) + ) + lines.append(f'transfer_output_remaps = "{_escape_macros(remaps)}"') + + return lines + + +def _render_resources(plan: SubmitPlan) -> list[str]: + lines = [f"request_cpus = {plan.cpus}", f"+xcount = {plan.cpus}"] + + if plan.memory_mb is not None: + lines.extend( + [f"request_memory = {plan.memory_mb}", f"+maxMemory = {plan.memory_mb}"] + ) + + if plan.wall_time_s is not None: + lines.append(f"+BatchRuntime = {plan.wall_time_s}") + + if plan.queue is not None: + lines.append(f"+queue = {_classad_string(plan.queue)}") + + return lines + + +def _render_queue(plan: SubmitPlan) -> list[str]: + return [ + # Keep completed jobs until purge() releases them + "leave_in_queue = (JobStatus == 4)", + f"queue {len(plan.copies)}", + ] diff --git a/src/intercede/_backends/credentials.py b/src/intercede/_backends/credentials.py new file mode 100644 index 0000000..02e804a --- /dev/null +++ b/src/intercede/_backends/credentials.py @@ -0,0 +1,157 @@ +"""What a backend asks its credential provider for, when, and what it accepts back. + +``credential_requirements`` turns a configuration into what to ask for. This is +also the one place freshness is decided. A backend keeps the credential it was +last given, privately, and before each operation calls ``refresh_if_stale``: +the provider is asked only when there is none yet or it is about to expire. +Where credentials come from, and how they are renewed, stays the consumer's +business, inside its provider. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from intercede.auth.models import ( + BearerToken, + Credential, + CredentialRequirements, + SSHPassword, + SSHPrivateKey, +) +from intercede.auth.protocols import CredentialProvider +from intercede.config import ( + ARCConfig, + BackendConfig, + BatchSSHConfig, + HTCondorCEConfig, +) +from intercede.exceptions import AuthenticationError, InterCEdeError + +# What each configured auth mode asks the credential provider for. +# "agent" is absent on purpose: an SSH agent holds the key, so nothing is asked. +AUTH_KINDS: dict[str, type[Credential]] = { + "token": BearerToken, + "key": SSHPrivateKey, + "password": SSHPassword, +} + +# Ask again this long before a credential expires, so it cannot run out in the +# middle of an operation. A library default; whether a provider should be able +# to change it is still open. +DEFAULT_MARGIN = timedelta(minutes=5) + + +def credential_requirements( + config: BackendConfig, scopes: frozenset[str] = frozenset() +) -> CredentialRequirements | None: + """Say what a backend must ask its provider for. + + Args: + config: The backend's validated configuration. + scopes: The scopes of the operation about to run, for token-based + backends; ignored by the others. + + Returns: + What to ask for, or None when the backend needs no credential: it runs + on this host, or an SSH agent holds the key. + """ + if isinstance(config, ARCConfig | HTCondorCEConfig): + # The endpoint is the resource, so a token is minted for this CE's audience. + return CredentialRequirements( + resource=config.endpoint, + kind=AUTH_KINDS[config.auth], + scopes=scopes, + ) + if isinstance(config, BatchSSHConfig) and config.auth != "agent": + # The resource is the login, ``user@host:port``: a provider finds the key + # or password by it, and the same host can take different users. + return CredentialRequirements( + resource=f"{config.user}@{config.host}:{config.port}", + kind=AUTH_KINDS[config.auth], + ) + return None + + +def is_stale( + credential: Credential | None, + requirements: CredentialRequirements, + *, + margin: timedelta = DEFAULT_MARGIN, + now: datetime | None = None, +) -> bool: + """Tell whether ``credential`` cannot serve ``requirements`` for ``margin`` more. + + A credential that never expires is never stale. + + Args: + credential: The one the backend holds, or None before its first request. + requirements: What the backend needs for the operation about to run. + margin: How long the credential must stay valid from now. + now: The current time; tests pass a fixed one. + + Returns: + True when there is no credential yet, when it is of another kind than + required, or when it expires within ``margin``. + """ + if not isinstance(credential, requirements.kind): + return True + if credential.expires_at is None: + return False + return credential.expires_at - margin <= (now or datetime.now(UTC)) + + +async def refresh_if_stale( + current: Credential | None, + requirements: CredentialRequirements, + provider: CredentialProvider, + *, + margin: timedelta = DEFAULT_MARGIN, + now: datetime | None = None, +) -> Credential: + """Return ``current`` while it is fresh, otherwise a new one from ``provider``. + + Args: + current: The credential the backend holds, or None before its first request. + requirements: What the backend needs for the operation about to run. + provider: The consumer's provider, asked only when ``current`` is stale. + margin: How long the credential must stay valid from now. + now: The current time; tests pass a fixed one. + + Returns: + The credential to authenticate with: ``current``, or the provider's new one. + + Raises: + AuthenticationError: The provider failed, returned something other than + the required kind, or returned a credential that has already expired. + Always a failure of the whole operation, never a per-job outcome. + """ + if not is_stale(current, requirements, margin=margin, now=now): + assert current is not None # is_stale() is True for None + return current + + try: + fresh = await provider.get(requirements) + except InterCEdeError: + raise # already typed, e.g. an AuthenticationError from the provider + except Exception as exc: + # Only the type: the provider's message may hold a token or a secret URL. + # The exception itself stays available as __cause__. + raise AuthenticationError( + f"the credential provider failed for {requirements.resource} " + f"with {type(exc).__name__}" + ) from exc + + if not isinstance(fresh, requirements.kind): + raise AuthenticationError( + f"the credential provider returned a {type(fresh).__name__} " + f"for {requirements.resource}, not a {requirements.kind.__name__}" + ) + # Expiring within the margin is accepted (it is asked again next time); + # already expired is not, since it cannot authenticate anything. + if is_stale(fresh, requirements, margin=timedelta(0), now=now): + raise AuthenticationError( + f"the credential provider returned an expired credential " + f"for {requirements.resource}" + ) + return fresh diff --git a/src/intercede/_backends/fake.py b/src/intercede/_backends/fake.py index 53c4d59..3171866 100644 --- a/src/intercede/_backends/fake.py +++ b/src/intercede/_backends/fake.py @@ -13,6 +13,7 @@ from typing import Self from intercede._backends.config import FakeConfig +from intercede.auth.protocols import CredentialProvider from intercede.exceptions import ( InterCEdeError, ResourceUnavailableError, @@ -94,11 +95,15 @@ def __init__( self._closed = False @classmethod - def from_settings(cls, settings: FakeConfig) -> Self: + def from_settings( + cls, settings: FakeConfig, credentials: CredentialProvider + ) -> Self: """Build the fake from its settings; how the registry constructs it. Args: settings: The fake's settings: only the resource it reports. + credentials: Never asked: nothing leaves the process, so the fake + needs no credentials. Returns: A new fake, not yet opened. @@ -307,7 +312,12 @@ def _read_local_inputs(inputs: Sequence[FileRef]) -> dict[str, bytes]: contents: dict[str, bytes] = {} for ref in inputs: if not _is_url(ref.source): - contents[ref.name] = Path(ref.source).read_bytes() + try: + contents[ref.name] = Path(ref.source).read_bytes() + except OSError as exc: + raise SpecificationRejectedError( + f"input {ref.name!r}: cannot read {ref.source}: {exc.strerror}" + ) from exc return contents @staticmethod @@ -379,4 +389,6 @@ def _write_job_files( _kill: type[Killable] = FakeBackend _purge: type[Purgeable] = FakeBackend _diag: type[Diagnosable] = FakeBackend -_built_by_registry: Callable[[FakeConfig], JobBackend] = FakeBackend.from_settings +_built_by_registry: Callable[[FakeConfig, CredentialProvider], JobBackend] = ( + FakeBackend.from_settings +) diff --git a/src/intercede/_backends/monolith/__init__.py b/src/intercede/_backends/monolith/__init__.py new file mode 100644 index 0000000..1a90eaa --- /dev/null +++ b/src/intercede/_backends/monolith/__init__.py @@ -0,0 +1 @@ +"""Monolithic backends: services that submit and schedule by themselves.""" diff --git a/src/intercede/_backends/monolith/htcondorce/__init__.py b/src/intercede/_backends/monolith/htcondorce/__init__.py new file mode 100644 index 0000000..21f0d5b --- /dev/null +++ b/src/intercede/_backends/monolith/htcondorce/__init__.py @@ -0,0 +1,23 @@ +"""An HTCondor-CE, reached as a remote schedd (issue #23). + +The backend drives the HTCondor command-line tools, as DIRAC's +``HTCondorCEComputingElement`` does with ``UseLocalSchedd = False``: jobs are +submitted with ``condor_submit -remote`` and their inputs spooled to the CE at +submit time, so no HTCondor daemon runs on the submitting host. + +- ``backend``: the contract operations (submit, status, output, kill, purge) +- ``session``: one operation's credentials, and the tools run with them +- ``staging``: the private working area, and what a submission puts in it +- ``collect``: moving collected output safely into ``dest`` +- ``process``: running one tool as a subprocess + +What HTCondor itself knows (submit descriptions, commands, the state mapping) +lives in ``_backends._htcondor`` and is shared with the HTCondor scheduler of +the composed backend. This package holds only what is specific to a remote CE. +""" + +from __future__ import annotations + +from intercede._backends.monolith.htcondorce.backend import HTCondorCEBackend + +__all__ = ["HTCondorCEBackend"] diff --git a/src/intercede/_backends/monolith/htcondorce/backend.py b/src/intercede/_backends/monolith/htcondorce/backend.py new file mode 100644 index 0000000..ab57d39 --- /dev/null +++ b/src/intercede/_backends/monolith/htcondorce/backend.py @@ -0,0 +1,412 @@ +"""The contract operations of the HTCondor-CE backend.""" + +from __future__ import annotations + +import asyncio +import re +import shutil +import uuid +import weakref +from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence +from contextlib import asynccontextmanager +from pathlib import Path +from types import TracebackType +from typing import Any, Self + +from intercede._backends._htcondor import status +from intercede._backends._htcondor.submit import SubmitPlan, render +from intercede._backends.credentials import credential_requirements, refresh_if_stale +from intercede._backends.monolith.htcondorce import collect, staging +from intercede._backends.monolith.htcondorce.process import Runner, run_process +from intercede._backends.monolith.htcondorce.session import ( + CondorSession, + credential_files, + failure, +) +from intercede.auth.models import BearerToken +from intercede.auth.protocols import CredentialProvider +from intercede.config import HTCondorCEConfig +from intercede.exceptions import InterCEdeError, SpecificationRejectedError +from intercede.models import ( + CopyVars, + JobHandle, + JobID, + JobOutput, + JobStatus, + OpOutcome, + StatusMap, + Submission, + SubmissionSpec, +) +from intercede.protocols import JobBackend, Killable, OutputRetriever, Purgeable + +BACKEND = "htcondor-ce" +DEFAULT_PORT = 9619 +PARALLEL_TRANSFERS = 4 + +# Token scopes per operation (WLCG token profile), so a status poller carries +# a read scope and nothing that would let it submit. +SUBMIT_SCOPES = frozenset({"compute.create"}) +READ_SCOPES = frozenset({"compute.read"}) +CANCEL_SCOPES = frozenset({"compute.cancel"}) + +_ACTIVE = frozenset({JobStatus.SUBMITTED, JobStatus.WAITING, JobStatus.RUNNING}) +_JOB_ID = re.compile(r"^\d+\.\d+$") +# Attributes get_output() needs on top of the state. +_OUTPUT_ATTRIBUTES = (*status.STATE_ATTRIBUTES, "Out", "Err", "UserLog", "SUBMIT_Iwd") + + +class HTCondorCEBackend: + """An HTCondor-CE: submit, status, output, kill and purge. + + It has no live diagnostics (issue #23); ``Diagnosable`` is absent on purpose. + """ + + def __init__( + self, + settings: HTCondorCEConfig, + credentials: CredentialProvider, + *, + runner: Runner = run_process, + workdir: Path | None = None, + ) -> None: + # Construction is cheap and does no I/O. + self._settings = settings + self._provider = credentials + self._run = runner + self._workdir = workdir or staging.default_workdir() + self._host, port = split_endpoint(settings.endpoint) + self._pool = f"{self._host}:{port}" + # The CE's schedd is named after its host unless the site says otherwise. + self._name = settings.schedd or self._host + self._tokens: dict[frozenset[str], BearerToken] = {} # per scope set + # Only jobs being collected right now: a lock disappears with its last + # user, so the backend keeps no record of the jobs it has seen. + self._collecting: weakref.WeakValueDictionary[JobID, asyncio.Lock] = ( + weakref.WeakValueDictionary() + ) + self._closed = False + + @classmethod + def from_settings( + cls, settings: HTCondorCEConfig, credentials: CredentialProvider + ) -> Self: + """Build the backend from its settings; how the registry constructs it. + + Args: + settings: The CE's validated configuration. + credentials: The consumer's provider, asked for a token when an + operation first needs one. + + Returns: + A new backend, not yet opened. + """ + return cls(settings, credentials) + + @property + def resource(self) -> str: + """The CE this backend talks to; the ``resource`` of its job handles.""" + return self._settings.endpoint + + # -- lifecycle --------------------------------------------------------- + + async def __aenter__(self) -> Self: + self._closed = False + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + # Idempotent. Cached tokens are dropped; they were never written anywhere. + self._closed = True + self._tokens.clear() + + # -- Submitter --------------------------------------------------------- + + async def submit( + self, spec: SubmissionSpec, copies: int | Sequence[CopyVars] = 1 + ) -> Submission: + self._check_usable() + copy_vars = [CopyVars()] * copies if isinstance(copies, int) else list(copies) + if not copy_vars: + return Submission(handles={}) + for index, copy in enumerate(copy_vars): + # A secret and an input would be staged to the same file. + if clashes := spec.clashes_with(copy): + raise SpecificationRejectedError( + f"copy {index}: secrets {sorted(clashes)} are named like inputs" + ) + outputs, remaps = staging.outputs(spec) + executable, ship, arguments = staging.command( + spec, self._settings.container_launcher, self.resource + ) + + handles: dict[int, JobHandle] = {} + failures: dict[int, str] = {} + async with self._session(SUBMIT_SCOPES) as session: + submission = self._workdir / uuid.uuid4().hex + submission.mkdir(mode=0o700) + try: + inputs = staging.stage_inputs(spec, submission) + for group in staging.group_by_environment(spec, copy_vars): + plan = SubmitPlan( + executable=executable, + transfer_executable=ship, + arguments=arguments, + copies=[ + staging.stage_copy(spec, copy_vars[i], submission / str(i)) + for i in group + ], + inputs=inputs, + stdout=spec.stdout or SubmitPlan.stdout, + stderr=spec.stderr or SubmitPlan.stderr, + outputs=outputs, + output_remaps=remaps, + cpus=spec.resources.cpus, + memory_mb=spec.resources.memory_mb, + wall_time_s=spec.resources.wall_time_s, + queue=spec.resources.queue or self._settings.queue, + ) + description = submission / f"group-{group[0]}.sub" + description.write_text(render(plan)) + try: + ids = await session.submit(description, len(group)) + except InterCEdeError as exc: + if not handles: + raise # nothing submitted: the whole operation failed + failures.update({i: str(exc) for i in group}) + continue + for index, job_id in zip(group, ids): + handles[index] = JobHandle( + backend=BACKEND, resource=self.resource, id=job_id + ) + finally: + # Inputs are spooled to the CE by now; secrets must not linger here. + shutil.rmtree(submission, ignore_errors=True) + return Submission(handles=handles, failures=failures) + + # -- StatusReporter ---------------------------------------------------- + + async def get_status(self, handles: Sequence[JobHandle]) -> StatusMap: + self._check_usable() + ads = await self._ads(self._ids(handles), status.STATE_ATTRIBUTES, history=True) + return { + handle: status.job_status(ads[handle.id]) + if self._owns(handle) and handle.id in ads + else JobStatus.UNKNOWN + for handle in handles + } + + # -- OutputRetriever --------------------------------------------------- + + async def get_output( + self, handles: Sequence[JobHandle], dest: Path + ) -> Mapping[JobHandle, OpOutcome[JobOutput]]: + self._check_usable() + ids = self._ids(handles) + queued = await self._ads(ids, _OUTPUT_ATTRIBUTES, history=False) + gone = await self._ads( + [i for i in ids if i not in queued], status.STATE_ATTRIBUTES, history=True + ) + + results: dict[JobHandle, OpOutcome[JobOutput]] = {} + ready: list[tuple[JobHandle, Mapping[str, Any]]] = [] + for handle in handles: + ad = queued.get(handle.id) if self._owns(handle) else None + if ad is not None and status.is_removed(ad): + # Purged a moment ago: released, though not out of the queue yet. + results[handle] = OpOutcome[JobOutput]( + ok=False, reason="the job was purged; its output is gone" + ) + elif ad is None: + known = self._owns(handle) and handle.id in gone + reason = ( + "the job has left the queue; its output is gone" + if known + else "unknown job" + ) + results[handle] = OpOutcome[JobOutput](ok=False, reason=reason) + elif status.job_status(ad) in _ACTIVE: + results[handle] = OpOutcome[JobOutput]( + ok=False, reason="job has not finished" + ) + else: + ready.append((handle, ad)) + + if ready: + limit = asyncio.Semaphore(PARALLEL_TRANSFERS) + async with self._session(READ_SCOPES) as session: + + async def fetch(handle: JobHandle, ad: Mapping[str, Any]) -> None: + async with limit: + results[handle] = await self._collect(session, handle, ad, dest) + + await asyncio.gather(*(fetch(handle, ad) for handle, ad in ready)) + return {handle: results[handle] for handle in handles} # in the caller's order + + async def _collect( + self, + session: CondorSession, + handle: JobHandle, + ad: Mapping[str, Any], + dest: Path, + ) -> OpOutcome[JobOutput]: + iwd = Path(str(ad.get("SUBMIT_Iwd", ""))) + # SUBMIT_Iwd comes from the remote job ad: never let the CE choose + # where on this host we write. + if not collect.inside_workdir(iwd, self._workdir): + return OpOutcome[JobOutput]( + ok=False, + reason="the job's output directory is outside our working area", + ) + # Two collections of one job would write into the same directory. + lock = self._collecting.get(handle.id) + if lock is None: + lock = self._collecting[handle.id] = asyncio.Lock() + async with lock: + try: + result = await session.transfer( + handle.id, self._settings.transfer_timeout_s + ) + if result.returncode != 0: + return OpOutcome[JobOutput]( + ok=False, reason=failure(result, "condor_transfer_data") + ) + target = dest / self._host / handle.id + return collect.move_output(iwd, ad, target, self._settings) + finally: + shutil.rmtree(iwd, ignore_errors=True) + + # -- Killable / Purgeable ---------------------------------------------- + + async def kill( + self, handles: Sequence[JobHandle] + ) -> Mapping[JobHandle, OpOutcome[None]]: + self._check_usable() + ids = self._ids(handles) + ads = await self._ads(ids, status.STATE_ATTRIBUTES, history=True) + # Only active jobs are removed: removing a finished job would release + # output that has not been collected yet. That is purge()'s job. + active = [i for i in ids if i in ads and status.job_status(ads[i]) in _ACTIVE] + removed, reason = await self._remove(active) + return { + handle: self._outcome( + handle, ads, active, removed, reason, "already finished" + ) + for handle in handles + } + + async def purge( + self, handles: Sequence[JobHandle] + ) -> Mapping[JobHandle, OpOutcome[None]]: + self._check_usable() + ids = self._ids(handles) + queued = await self._ads(ids, status.STATE_ATTRIBUTES, history=False) + gone = await self._ads( + [i for i in ids if i not in queued], status.STATE_ATTRIBUTES, history=True + ) + # A job removed a moment ago is still queued, but already released. + targeted = [i for i, ad in queued.items() if not status.is_removed(ad)] + removed, reason = await self._remove(targeted) + ads = {**gone, **queued} + return { + handle: self._outcome( + handle, ads, targeted, removed, reason, "already released" + ) + for handle in handles + } + + def _outcome( + self, + handle: JobHandle, + ads: Mapping[JobID, Mapping[str, Any]], + targeted: Sequence[JobID], + removed: set[JobID], + reason: str, + untouched: str, + ) -> OpOutcome[None]: + if not self._owns(handle) or handle.id not in ads: + return OpOutcome[None](ok=False, reason="unknown job") + if handle.id not in targeted: + return OpOutcome[None](ok=True, reason=untouched) + if handle.id in removed: + return OpOutcome[None](ok=True) + return OpOutcome[None]( + ok=False, reason=reason or "condor_rm did not remove the job" + ) + + # -- talking to the CE ---------------------------------------------------- + + @asynccontextmanager + async def _session(self, scopes: frozenset[str]) -> AsyncGenerator[CondorSession]: + """Open the tools for one operation, with a token scoped to it.""" + token = await self._fresh(scopes) + workdir = staging.ensure_workdir(self._workdir) + async with credential_files(token, workdir) as files: + yield CondorSession(self._run, files, self._pool, self._name) + + async def _fresh(self, scopes: frozenset[str]) -> BearerToken: + requirements = credential_requirements(self._settings, scopes) + assert requirements is not None # an HTCondor-CE always asks for a token + fresh = await refresh_if_stale( + self._tokens.get(scopes), requirements, self._provider + ) + assert isinstance(fresh, BearerToken) # refresh_if_stale checked the kind + self._tokens[scopes] = fresh + return fresh + + async def _ads( + self, ids: Sequence[JobID], attributes: Sequence[str], *, history: bool + ) -> dict[JobID, Mapping[str, Any]]: + if not ids: + return {} + async with self._session(READ_SCOPES) as session: + return await session.ads(ids, attributes, history=history) + + async def _remove(self, ids: Sequence[JobID]) -> tuple[set[JobID], str]: + if not ids: + return set(), "" + async with self._session(CANCEL_SCOPES) as session: + return await session.remove(ids) + + # -- helpers ------------------------------------------------------------- + + def _check_usable(self) -> None: + if self._closed: + raise InterCEdeError("backend is closed") + + def _owns(self, handle: JobHandle) -> bool: + return ( + handle.backend == BACKEND + and handle.resource == self.resource + and bool(_JOB_ID.match(handle.id)) + ) + + def _ids(self, handles: Iterable[JobHandle]) -> list[JobID]: + """Return the ids to ask the CE about: ours, well-formed, each once. + + Ids come from stored handles, so they are checked before they reach a + command line: a malformed one could otherwise pass as an option. + """ + return list( + dict.fromkeys(handle.id for handle in handles if self._owns(handle)) + ) + + +def split_endpoint(endpoint: str) -> tuple[str, int]: + """Split ``host[:port]``; an HTCondor-CE listens on 9619 by default.""" + host, separator, port = endpoint.partition(":") + return host, int(port) if separator else DEFAULT_PORT + + +# The type checker proves conformance. +_backend: type[JobBackend] = HTCondorCEBackend +_output: type[OutputRetriever] = HTCondorCEBackend +_kill: type[Killable] = HTCondorCEBackend +_purge: type[Purgeable] = HTCondorCEBackend +_built_by_registry: Callable[[HTCondorCEConfig, CredentialProvider], JobBackend] = ( + HTCondorCEBackend.from_settings +) diff --git a/src/intercede/_backends/monolith/htcondorce/collect.py b/src/intercede/_backends/monolith/htcondorce/collect.py new file mode 100644 index 0000000..4c5a129 --- /dev/null +++ b/src/intercede/_backends/monolith/htcondorce/collect.py @@ -0,0 +1,122 @@ +"""Moving collected output from a job's directory into ``dest``. + +``condor_transfer_data`` writes into the directory a job was submitted from, +recorded as ``SUBMIT_Iwd`` in the job ad. That path comes from the CE, so it +is only trusted inside our own working area, and what lands there is checked +against the configured limits before anything reaches ``dest``. +""" + +from __future__ import annotations + +import shutil +from collections.abc import Mapping +from pathlib import Path, PurePosixPath +from typing import Any + +from intercede.config import HTCondorCEConfig +from intercede.models import JobOutput, OpOutcome + + +def inside_workdir(iwd: Path, workdir: Path) -> bool: + """Tell whether a job directory lies strictly inside our working area.""" + root = workdir.resolve() + return ( + iwd.is_absolute() + and iwd.resolve().is_relative_to(root) + and iwd.resolve() != root + ) + + +def move_output( + iwd: Path, ad: Mapping[str, Any], target: Path, limits: HTCondorCEConfig +) -> OpOutcome[JobOutput]: + """Move a job's collected files to ``target`` and describe them in a manifest.""" + if not iwd.is_dir(): + return _failed("condor_transfer_data returned no files") + + found = _find_output_files(iwd) + error = _validate_output_files(iwd, found, limits) + if error is not None: + return _failed(error) + + _prepare_output_directory(target) + manifest = _move_output_files(found, iwd, target, ad) + + return OpOutcome[JobOutput](ok=True, value=JobOutput(**manifest)) + + +def relative(value: object, iwd: Path) -> str | None: + """Return the path of an ad's file attribute relative to the job directory. + + ``Out`` and ``Err`` are relative to it already; ``UserLog`` is absolute. + """ + if not isinstance(value, str) or not value: + return None + path = PurePosixPath(value) + if not path.is_absolute(): + return path.as_posix() + try: + return path.relative_to(PurePosixPath(iwd)).as_posix() + except ValueError: + return None + + +def _find_output_files(iwd: Path) -> list[Path]: + """Find non-directory entries in the collected output tree.""" + return [path for path in iwd.rglob("*") if not path.is_dir()] + + +def _validate_output_files( + iwd: Path, found: list[Path], limits: HTCondorCEConfig +) -> str | None: + """Return an error message if the collected files are unsafe or too large.""" + root = iwd.resolve() + + for path in found: + if path.is_symlink() or not path.resolve().is_relative_to(root): + return "the output holds a link or a path outside the job directory" + + if len(found) > limits.max_output_files: + return f"more than {limits.max_output_files} output files" + + total_bytes = sum(path.stat().st_size for path in found) + if total_bytes > limits.max_output_bytes: + return f"output larger than {limits.max_output_bytes} bytes" + + return None + + +def _prepare_output_directory(target: Path) -> None: + """Replace the previous collection with a fresh output directory.""" + shutil.rmtree(target, ignore_errors=True) + target.mkdir(parents=True, mode=0o700) + + +def _move_output_files( + found: list[Path], iwd: Path, target: Path, ad: Mapping[str, Any] +) -> dict[str, Any]: + """Move files into the destination and build the output manifest.""" + special = { + relative(ad.get("Out"), iwd): "stdout", + relative(ad.get("Err"), iwd): "stderr", + relative(ad.get("UserLog"), iwd): "log", + } + manifest: dict[str, Any] = {"files": {}} + + for path in found: + name = path.relative_to(iwd).as_posix() + destination = target / name + + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.move(path, destination) + + if name in special: + manifest[special[name]] = destination + else: + manifest["files"][name] = destination + + return manifest + + +def _failed(reason: str) -> OpOutcome[JobOutput]: + return OpOutcome[JobOutput](ok=False, reason=reason) diff --git a/src/intercede/_backends/monolith/htcondorce/process.py b/src/intercede/_backends/monolith/htcondorce/process.py new file mode 100644 index 0000000..b2db0ea --- /dev/null +++ b/src/intercede/_backends/monolith/htcondorce/process.py @@ -0,0 +1,44 @@ +"""Running one HTCondor tool as a subprocess.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable, Mapping, Sequence + +from intercede._backends.batch.protocols import CommandResult +from intercede.exceptions import ConfigurationError, ResourceUnavailableError + +# How the backend runs a tool: argv, environment, timeout in seconds. +# Tests pass their own, to play the tools without HTCondor installed. +Runner = Callable[[Sequence[str], Mapping[str, str], float], Awaitable[CommandResult]] + + +async def run_process( + argv: Sequence[str], env: Mapping[str, str], timeout: float +) -> CommandResult: + """Run one HTCondor tool and wait for it, killing it after ``timeout`` seconds.""" + try: + process = await asyncio.create_subprocess_exec( + *argv, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=dict(env), + ) + except FileNotFoundError as exc: + raise ConfigurationError( + f"{argv[0]} was not found: the HTCondor client tools must be installed" + ) from exc + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout) + except TimeoutError as exc: + process.kill() + await process.wait() + raise ResourceUnavailableError( + f"{argv[0]} did not finish within {timeout:.0f} s" + ) from exc + return CommandResult( + returncode=process.returncode if process.returncode is not None else -1, + stdout=stdout.decode(errors="replace"), + stderr=stderr.decode(errors="replace"), + ) diff --git a/src/intercede/_backends/monolith/htcondorce/session.py b/src/intercede/_backends/monolith/htcondorce/session.py new file mode 100644 index 0000000..cf9a388 --- /dev/null +++ b/src/intercede/_backends/monolith/htcondorce/session.py @@ -0,0 +1,233 @@ +"""One operation's connection to the CE: its credentials, and the tools run with them. + +Every tool runs in its own subprocess, configured only through its +environment. The token for an operation is written to private files that exist +only while the operation runs, so backends with different credentials never +share HTCondor configuration. +""" + +from __future__ import annotations + +import os +import re +import shutil +import tempfile +from collections.abc import AsyncGenerator, Mapping, Sequence +from contextlib import asynccontextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from intercede._backends._htcondor import commands, status +from intercede._backends.batch.protocols import CommandResult +from intercede._backends.monolith.htcondorce.process import Runner +from intercede._backends.monolith.htcondorce.staging import write_private +from intercede.auth.models import BearerToken +from intercede.exceptions import AuthenticationError, ResourceUnavailableError +from intercede.models import JobID + +COMMAND_TIMEOUT_S = 120 # every command but output transfer, which uses the limits + + +@dataclass(frozen=True) +class CredentialFiles: + """Where one operation's token is, while it runs.""" + + token: Path + token_dir: Path + scratch: Path # HTCondor's LOG directory (SciTokens keeps a cache there) + + +@asynccontextmanager +async def credential_files( + token: BearerToken, workdir: Path +) -> AsyncGenerator[CredentialFiles, None]: + """Write the token to private files for the duration of one operation.""" + directory = Path(tempfile.mkdtemp(prefix="cred-", dir=workdir)) # mode 0700 + try: + token_dir = directory / "tokens.d" + token_dir.mkdir(mode=0o700) + secret = token.value.get_secret_value().encode() + for path in (directory / "token", token_dir / "token"): + write_private(path, secret) + scratch = directory / "log" + scratch.mkdir(mode=0o700) + yield CredentialFiles( + token=directory / "token", + token_dir=token_dir, + scratch=scratch, + ) + finally: + shutil.rmtree(directory, ignore_errors=True) + + +def environment(files: CredentialFiles) -> dict[str, str]: + """Return the environment a tool runs with, for the credentials in ``files``.""" + env = dict(os.environ) + if "CONDOR_CONFIG" not in env: + # No config file: this environment is all the configuration there is. + env["CONDOR_CONFIG"] = "ONLY_ENV" + env.update( + { + # Never fall back to an unauthenticated connection: by default + # HTCondor would quietly continue anonymously after a rejected token. + "_CONDOR_SEC_CLIENT_AUTHENTICATION": "REQUIRED", + # SCITOKENS for a CE; IDTOKENS for a schedd that issues its own + # tokens. The same token is offered to both, in that order. + "_CONDOR_SEC_CLIENT_AUTHENTICATION_METHODS": "SCITOKENS,IDTOKENS", + "_CONDOR_SCITOKENS_FILE": str(files.token), + "_CONDOR_SEC_TOKEN_DIRECTORY": str(files.token_dir), + "_CONDOR_LOG": str(files.scratch), + "_CONDOR_TOOL_DEBUG": "D_SECURITY", + } + ) + if "X509_CERT_DIR" in env: # the CAs the CE's host certificate is checked against + env["_CONDOR_AUTH_SSL_CLIENT_CADIR"] = env["X509_CERT_DIR"] + return env + + +class CondorSession: + """The HTCondor tools, run against one CE with one operation's credentials.""" + + def __init__( + self, runner: Runner, files: CredentialFiles, pool: str, name: str + ) -> None: + self.files = files + self._run = runner + self._pool = pool + self._name = name + self._remote = commands.schedd_options(pool, name) + self._env = environment(files) + + async def run( + self, argv: Sequence[str], timeout: float = COMMAND_TIMEOUT_S + ) -> CommandResult: + """Run a tool; raise when the CE rejected us or could not be reached.""" + # -debug with D_SECURITY makes the tool say why a connection failed: + # the only way to tell a rejected token from an unreachable CE. + result = await self._run([argv[0], "-debug", *argv[1:]], self._env, timeout) + if result.returncode == 0: + return result + text = f"{result.stdout}\n{result.stderr}" + if rejected := _AUTH_FAILURE.search(text): + raise AuthenticationError( + f"{self._name} rejected the credentials: {_line_of(text, rejected)}" + ) + if _UNREACHABLE.search(text): + raise ResourceUnavailableError( + f"{self._pool} could not be reached: {last_line(result)}" + ) + return result + + async def submit(self, description: Path, expected: int) -> list[JobID]: + """Submit a description file and return the ids of the jobs it created.""" + result = await self.run( + commands.submit(str(description), self._pool, self._name) + ) + require_success(result, "condor_submit") + try: + ids = commands.parse_submit(result.stdout) + except ValueError as exc: + raise ResourceUnavailableError(str(exc)) from exc + if len(ids) != expected: + raise ResourceUnavailableError( + f"condor_submit created {len(ids)} jobs, expected {expected}" + ) + return ids + + async def ads( + self, ids: Sequence[JobID], attributes: Sequence[str], *, history: bool + ) -> dict[JobID, Mapping[str, Any]]: + """Read job ads: from the queue, then (with ``history``) the rest from the history.""" + found: dict[JobID, Mapping[str, Any]] = {} + for chunk in commands.chunks(ids): + result = await self._read(commands.query(chunk, attributes, self._remote)) + found.update(result) + if history: + missing = [i for i in ids if i not in found] + for chunk in commands.chunks(missing): + result = await self._read( + commands.history(chunk, attributes, self._remote) + ) + found.update(result) + return found + + async def remove(self, ids: Sequence[JobID]) -> tuple[set[JobID], str]: + """Remove jobs; return those removed, and why the others were not.""" + removed: set[JobID] = set() + reason = "" + for chunk in commands.chunks(ids): + result = await self.run(commands.remove(chunk, self._remote)) + parsed_result = commands.parse_remove(result.stdout) + removed.update(parsed_result) + if result.returncode != 0: + reason = failure(result, "condor_rm") + return removed, reason + + async def transfer(self, job_id: JobID, timeout: float) -> CommandResult: + """Fetch one job's spooled output into the directory it was submitted from.""" + return await self.run(commands.transfer_data([job_id], self._remote), timeout) + + async def _read(self, argv: Sequence[str]) -> dict[JobID, Mapping[str, Any]]: + result = await self.run(argv) + require_success(result, argv[0]) + try: + ads = status.parse_ads(result.stdout) + except ValueError as exc: # also covers json.JSONDecodeError + raise ResourceUnavailableError( + f"{argv[0]} printed unreadable output: {exc}" + ) from exc + return { + status.job_id(ad): ad for ad in ads if "ClusterId" in ad and "ProcId" in ad + } + + +# -- reading what a tool said --------------------------------------------------- + +# Markers in the -debug output. An authentication failure also reports that +# the collector "could not be contacted", so it is checked first. +_AUTH_FAILURE = re.compile( + # condor_submit prints only the SECMAN line; condor_q also the AUTHENTICATE + # summary. Not "method ... failed": that is logged even when the next + # method then succeeds. + r"required authentication with .* failed|AUTHENTICATE:\d+|PERMISSION DENIED|not authorized", + re.IGNORECASE, +) +_UNREACHABLE = re.compile( + r"Connection refused|connect errno|Failed to connect|Couldn't contact|Can't find address|" + r"couldn't locate|Unable to locate|Name or service not known|timed out", + re.IGNORECASE, +) +_DEBUG_LINE = re.compile(r"^\d\d/\d\d/\d\d \d\d:\d\d:\d\d ") + + +def _line_of(text: str, match: re.Match[str]) -> str: + """Return the line holding ``match``, without its -debug timestamp.""" + start = text.rfind("\n", 0, match.start()) + 1 + end = text.find("\n", match.end()) + line = text[start : end if end != -1 else len(text)].strip() + return _DEBUG_LINE.sub("", line) + + +def last_line(result: CommandResult) -> str: + """Return the tool's own last message, without the -debug lines.""" + lines = [ + line.strip() + for line in f"{result.stdout}\n{result.stderr}".splitlines() + if line.strip() and not _DEBUG_LINE.match(line) + ] + if lines: + return lines[-1] + else: + return f"exit code {result.returncode}" + + +def failure(result: CommandResult, tool: str) -> str: + """Describe why a tool failed, in one line.""" + return f"{tool} failed: {last_line(result)}" + + +def require_success(result: CommandResult, tool: str) -> None: + """Raise when a tool whose failure fails the whole operation did not succeed.""" + if result.returncode != 0: + raise ResourceUnavailableError(failure(result, tool)) diff --git a/src/intercede/_backends/monolith/htcondorce/staging.py b/src/intercede/_backends/monolith/htcondorce/staging.py new file mode 100644 index 0000000..79cccde --- /dev/null +++ b/src/intercede/_backends/monolith/htcondorce/staging.py @@ -0,0 +1,187 @@ +"""The private working area, and what a submission puts in it. + +Everything here turns a ``SubmissionSpec`` into what HTCondor is given: files +staged under their names, one directory per copy holding only that copy's +secrets, and the executable, arguments and outputs in HTCondor's terms. +""" + +from __future__ import annotations + +import os +import re +import stat +import tempfile +from collections.abc import Sequence +from pathlib import Path, PurePosixPath + +from intercede._backends._htcondor.submit import CopyPlan +from intercede.exceptions import ConfigurationError, SpecificationRejectedError +from intercede.models import CopyVars, SubmissionSpec + +_GLOB = re.compile(r"[*?\[]") + + +# -- the working area --------------------------------------------------------- + + +def default_workdir() -> Path: + """Return where this user's staging and collection areas live on this host.""" + return Path(tempfile.gettempdir()) / f"intercede-htcondor-ce-{os.getuid()}" + + +def ensure_workdir(workdir: Path) -> Path: + """Create the private working area, refusing one somebody else controls.""" + workdir.mkdir(mode=0o700, parents=True, exist_ok=True) + info = workdir.lstat() + if ( + stat.S_ISLNK(info.st_mode) + or not stat.S_ISDIR(info.st_mode) + or info.st_uid != os.getuid() + ): + raise ConfigurationError(f"{workdir} is not a directory owned by this user") + workdir.chmod(0o700) + return workdir + + +def write_private(path: Path, content: bytes) -> None: + """Create a file only this user can read (0600); never overwrite one.""" + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + + +# -- files -------------------------------------------------------------------- + + +def stage_inputs(spec: SubmissionSpec, submission: Path) -> list[str]: + """Copy local inputs under their names, so they spool with the right layout. + + Returns the entries for ``transfer_input_files``: the staged directory and + the URLs the resource fetches itself. + """ + staged: list[str] = [] + common = submission / "inputs" + for ref in spec.inputs: + if "://" in ref.source: + if PurePosixPath(ref.source.split("?")[0]).name != ref.name: + raise SpecificationRejectedError( + f"input {ref.name!r}: HTCondor names a URL input after the URL, " + f"so {ref.source!r} would arrive under another name" + ) + staged.append(ref.source) # fetched by the resource + continue + target = common / ref.name + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + # Read now: the caller may delete its file once submit() returns. + try: + content = Path(ref.source).read_bytes() + except OSError as exc: + raise SpecificationRejectedError( + f"input {ref.name!r}: cannot read {ref.source}: {exc.strerror}" + ) from exc + target.write_bytes(content) + if common.exists(): + # Trailing slash: the directory's contents, keeping sub-directories. + staged.append(f"{common}/") + return staged + + +def stage_copy(spec: SubmissionSpec, copy: CopyVars, directory: Path) -> CopyPlan: + """Give one copy its own directory, holding only its secrets (0600).""" + directory.mkdir(mode=0o700) + inputs: list[str] = [] + if copy.secrets: + secrets = directory / "secrets" + secrets.mkdir(mode=0o700) + for name, secret in copy.secrets.items(): + target = secrets / name + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + write_private(target, secret.get_secret_value()) # HTCondor keeps the mode + inputs.append("secrets/") + return CopyPlan( + initialdir=str(directory), + environment={**spec.environment, **copy.environment}, + inputs=inputs, + ) + + +def group_by_environment( + spec: SubmissionSpec, copies: Sequence[CopyVars] +) -> list[list[int]]: + """Group copy indexes by the names of their environment variables. + + HTCondor merges each process's environment onto the previous one, so a + variable of one copy would leak into the next in the same cluster: copies + with different names must go to different clusters. + """ + groups: dict[frozenset[str], list[int]] = {} + for index, copy in enumerate(copies): + names = frozenset({*spec.environment, *copy.environment}) + groups.setdefault(names, []).append(index) + return list(groups.values()) + + +# -- what HTCondor runs and returns --------------------------------------------- + + +def command( + spec: SubmissionSpec, launcher: Sequence[str] | None, resource: str +) -> tuple[str, bool, list[str]]: + """Return what HTCondor runs: the executable, whether to ship it, and its arguments. + + A file on this host is shipped; anything else is a command on the worker. + A container image is run through the configured launcher. + """ + executable = spec.executable + arguments = list(spec.arguments) + is_local_file = Path(spec.executable).is_file() + + if spec.container is None: + return executable, is_local_file, arguments + + if launcher is None: + raise SpecificationRejectedError( + f"{resource} has no container_launcher configured, so it cannot run " + f"{spec.container.image!r}" + ) + + if is_local_file: + raise SpecificationRejectedError( + "a container job runs a command inside the image, not a local file" + ) + + launcher_args = [arg.replace("{image}", spec.container.image) for arg in launcher] + + return (launcher_args[0], False, [*launcher_args[1:], executable, *arguments]) + + +def outputs(spec: SubmissionSpec) -> tuple[list[str] | None, dict[str, str]]: + """Return output files to retrieve and files to remap. + + None means retrieve all outputs. Remaps map output names to destinations. + """ + if spec.outputs is None: + return [], {} + + include_unlisted = spec.outputs.include_unlisted + kept: list[str] = [] + remaps: dict[str, str] = {} + + for member in spec.outputs.members: + pattern = member.pattern + destination = member.destination + + if _GLOB.search(pattern): + if not include_unlisted or destination is not None: + raise SpecificationRejectedError( + f"output {pattern!r}: " + "HTCondor selects outputs by name, not by pattern" + ) + continue # include_unlisted brings it back anyway + + kept.append(pattern) + + if destination is not None: + remaps[pattern] = destination # uploaded by the resource + + return (None if include_unlisted else kept), remaps diff --git a/src/intercede/auth/__init__.py b/src/intercede/auth/__init__.py new file mode 100644 index 0000000..0b886c3 --- /dev/null +++ b/src/intercede/auth/__init__.py @@ -0,0 +1,28 @@ +"""Backend credentials: typed, declared, supplied by a provider. + +A consumer implements a ``CredentialProvider``, which returns the one credential +a backend asks for. Configuration never holds a secret: it only says how a backend +logs in, and the provider supplies the credential at run time. +""" + +from __future__ import annotations + +from intercede.auth.models import ( + BearerToken, + Credential, + CredentialRequirements, + SSHPassword, + SSHPrivateKey, +) +from intercede.auth.protocols import CredentialProvider +from intercede.auth.providers import StaticCredentials + +__all__ = [ + "BearerToken", + "Credential", + "CredentialProvider", + "CredentialRequirements", + "SSHPassword", + "SSHPrivateKey", + "StaticCredentials", +] diff --git a/src/intercede/auth/models.py b/src/intercede/auth/models.py new file mode 100644 index 0000000..868c5b9 --- /dev/null +++ b/src/intercede/auth/models.py @@ -0,0 +1,106 @@ +"""The public credential models a consumer of interCEde can use. + +Credentials are immutable values. Secret material is held in pydantic's secret +types, so a credential cannot be printed or logged by accident: ``repr``, +``str`` and JSON all show ``**********``, and the content is only readable on +purpose, with ``get_secret_value()``. +""" + +from __future__ import annotations + +from typing import Self + +from pydantic import ( + AwareDatetime, + ConfigDict, + SecretBytes, + SecretStr, + model_validator, +) + +from intercede.models import _Model + +__all__ = [ + "BearerToken", + "Credential", + "CredentialRequirements", + "SSHPassword", + "SSHPrivateKey", +] + + +# -------------------------------------------------------------------------- +# Credential values +# -------------------------------------------------------------------------- + + +class Credential(_Model): + """Base of every credential: the one field the freshness check needs. + + Not used on its own: a provider returns one of the concrete kinds below. + """ + + # Added to _Model's settings: a value given by mistake may be the secret + # itself, so no error message repeats one. + model_config = ConfigDict(hide_input_in_errors=True, use_attribute_docstrings=True) + + expires_at: AwareDatetime | None = None + """When the credential stops working, with a timezone, so it can be compared + with the current time. None means it never expires, as for SSH secrets.""" + + +class BearerToken(Credential): + """A token for the channel to a CE. Opaque to interCEde.""" + + value: SecretStr + """The token itself.""" + + +class SSHPrivateKey(Credential): + """An SSH private key: the key itself, never a path on some host.""" + + pem: SecretBytes + """The private key, PEM-encoded.""" + + passphrase: SecretStr | None = None + """The passphrase the key is encrypted with, or None if it is not.""" + + +class SSHPassword(Credential): + """A password for an SSH login.""" + + value: SecretStr + """The password itself.""" + + +# -------------------------------------------------------------------------- +# What a backend asks for +# -------------------------------------------------------------------------- + + +class CredentialRequirements(_Model): + """What a backend asks its provider for. + + The backend computes it from its own configuration and keeps it to itself: + a consumer only meets it as the argument to its provider's ``get``. + """ + + model_config = ConfigDict(use_attribute_docstrings=True) + + resource: str + """Which backend instance is asking: the endpoint it talks to, the same + string as ``JobHandle.resource``. For a token this is also the audience, + in the form that CE expects.""" + + kind: type[Credential] + """The kind of credential needed: a backend authenticates with exactly one.""" + + scopes: frozenset[str] = frozenset() + """For a token, the scopes of the operation about to run; empty for the + other kinds.""" + + @model_validator(mode="after") + def _concrete_kind(self) -> Self: + if self.kind is Credential: + raise ValueError("kind must be a concrete kind, not the Credential base") + return self diff --git a/src/intercede/auth/protocols.py b/src/intercede/auth/protocols.py new file mode 100644 index 0000000..f3b9951 --- /dev/null +++ b/src/intercede/auth/protocols.py @@ -0,0 +1,48 @@ +"""The public protocol for a ``CredentialProvider``.""" + +from __future__ import annotations + +from types import TracebackType +from typing import Protocol, Self, runtime_checkable + +from intercede.auth.models import Credential, CredentialRequirements + +__all__ = ["CredentialProvider"] + + +@runtime_checkable +class CredentialProvider(Protocol): + """The protocol any consumer's own credential provider must satisfy. + + The consumer creates the provider, shares it between backends and closes + it: interCEde never closes a provider, because it did not create it. A + backend calls ``get`` only when its current credential is missing or + close to expiry, so the consumer decides where credentials come from and + interCEde decides when to ask. + """ + + async def get(self, requirements: CredentialRequirements) -> Credential: + """Return the credential a backend needs. + + Args: + requirements: The resource, the kind of credential it takes, and + the scopes of the operation about to run. + + Returns: + One credential of ``requirements.kind``. + + Raises: + Exception: Anything raised here fails the whole operation: the + backend reports it as an ``AuthenticationError``, never as a + per-job outcome. + """ + ... + + async def __aenter__(self) -> Self: ... + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: ... diff --git a/src/intercede/auth/providers.py b/src/intercede/auth/providers.py new file mode 100644 index 0000000..f6c2fe1 --- /dev/null +++ b/src/intercede/auth/providers.py @@ -0,0 +1,77 @@ +"""Ready-made credential providers.""" + +from __future__ import annotations + +from collections.abc import Iterable +from types import TracebackType +from typing import Self + +from intercede.auth.models import Credential, CredentialRequirements +from intercede.auth.protocols import CredentialProvider +from intercede.exceptions import AuthenticationError + +__all__ = ["StaticCredentials"] + + +class StaticCredentials: + """A provider holding fixed credentials: files read once, or fixed values. + + For standalone use, tests and the integration stacks, where nothing issues + or renews credentials. It holds at most one credential of each kind, since + one provider can serve several backends, and each backend receives only the + one it asks for. It holds nothing to release, so closing does nothing, but + it is still a context manager, like every provider:: + + async with StaticCredentials([SSHPrivateKey(pem=key)]) as credentials: + backend = registry.get_backend(config, credentials) + """ + + def __init__(self, credentials: Iterable[Credential] = ()) -> None: + # Validated now, so a duplicate kind or plain data fails at construction. + self._credentials: dict[type[Credential], Credential] = {} + for credential in credentials: + # A plain dict is ambiguous: {"value": ...} fits both a BearerToken + # and an SSHPassword. Guessing the kind of a secret is never right. + # Never repeat the item in the message: it may be the secret itself. + if not isinstance(credential, Credential) or type(credential) is Credential: + raise ValueError( + "StaticCredentials holds concrete credential objects, " + "e.g. BearerToken(value=...), not plain data" + ) + kind = type(credential) + if kind in self._credentials: + raise ValueError(f"more than one credential of kind {kind.__name__}") + self._credentials[kind] = credential + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + return None + + async def get(self, requirements: CredentialRequirements) -> Credential: + """Return the held credential of the requested kind. + + Args: + requirements: What the backend asks for. + + Returns: + The held credential of ``requirements.kind``. + + Raises: + AuthenticationError: It holds no credential of that kind. + """ + credential = self._credentials.get(requirements.kind) + if credential is None: + raise AuthenticationError( + f"no {requirements.kind.__name__} for {requirements.resource}" + ) + return credential + + +_provider: type[CredentialProvider] = StaticCredentials diff --git a/src/intercede/config.py b/src/intercede/config.py index e9be28a..f2a9c7b 100644 --- a/src/intercede/config.py +++ b/src/intercede/config.py @@ -16,13 +16,14 @@ known_hosts = "batch.example.org ssh-ed25519 AAAA..." Configuration describes the resource and is never secret: keys, passwords, -tokens and proxies come from the credential provider at run time. Error +and tokens come from the credential provider at run time. Error messages therefore never repeat the values they were given. The kinds are a closed set: they list what interCEde ships. """ from __future__ import annotations +import re from collections.abc import Mapping, Sequence from typing import Annotated, Any, Literal, Self @@ -91,6 +92,10 @@ def _launcher_names_the_image(self) -> Self: # -- grid CEs that schedule by themselves ------------------------------------- +_HOST_PORT = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?(?::\d{1,5})?") +# "host" or "name@host": no leading "-" and no spaces, so never read as an option. +_SCHEDD = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*(?:@[A-Za-z0-9][A-Za-z0-9.-]*)?") + class ARCConfig(_QueueConfig): """An ARC CE.""" @@ -101,15 +106,8 @@ class ARCConfig(_QueueConfig): endpoint: str """The CE's REST endpoint, e.g. ``"https://arc.example.org:443/arex"``.""" - auth: frozenset[Literal["token", "proxy"]] = frozenset({"token"}) - """Which credentials the provider must supply, all of them together: a token - for the channel, a proxy for delegation, or both.""" - - @model_validator(mode="after") - def _auth_not_empty(self) -> Self: - if not self.auth: - raise ValueError("auth must name at least one of 'token', 'proxy'") - return self + auth: Literal["token"] = "token" + """Which credential the provider must supply: a token for the channel.""" class HTCondorCEConfig(_QueueConfig): @@ -121,15 +119,23 @@ class HTCondorCEConfig(_QueueConfig): endpoint: str """The CE's collector, as ``"host"`` or ``"host:port"``, e.g. ``"ce.example.org:9619"``.""" - auth: frozenset[Literal["token", "proxy"]] = frozenset({"token"}) - """Which credentials the provider must supply. The token authenticates the - channel; ``"proxy"`` adds an X.509 proxy for sites that account on VOMS attributes.""" + schedd: str | None = None + """The CE's schedd, when it is not named after the endpoint's host: ``"name@host"`` + for one of several CEs on one host, or a CE behind a forwarded address.""" + + auth: Literal["token"] = "token" + """Which credential the provider must supply: a token for the channel.""" @model_validator(mode="after") - def _token_required(self) -> Self: - if "token" not in self.auth: + def _names_fit_a_command_line(self) -> Self: + # Both end up on the tools' command lines, as "-pool host:port -name schedd". + if not _HOST_PORT.fullmatch(self.endpoint): + raise ValueError( + "endpoint must be 'host' or 'host:port', e.g. 'ce.example.org:9619'" + ) + if self.schedd is not None and not _SCHEDD.fullmatch(self.schedd): raise ValueError( - "an HTCondor-CE authenticates with a token; auth must include 'token'" + "schedd must be 'host' or 'name@host', e.g. 'ce01@ce.example.org'" ) return self diff --git a/src/intercede/registry.py b/src/intercede/registry.py index 063afe1..9c57db7 100644 --- a/src/intercede/registry.py +++ b/src/intercede/registry.py @@ -7,6 +7,8 @@ Every registered class provides a ``from_settings`` classmethod, which is how the registry builds it from a validated configuration model. +The parts that authenticate (single-class backends and transports) also +receive the consumer's credential provider; a scheduler only builds commands. What it builds is checked against the relevant protocol before it is returned, so a broken entry fails here, with a clear error, and not at the first call. """ @@ -20,6 +22,7 @@ from intercede._backends.batch.protocols import Scheduler, Transport from intercede._backends.config import FakeConfig +from intercede.auth.protocols import CredentialProvider from intercede.config import ( _MODELS as _CONFIG_MODELS, ) @@ -29,7 +32,7 @@ BatchSSHConfig, validate_config, ) -from intercede.exceptions import RegistryError +from intercede.exceptions import ConfigurationError, RegistryError from intercede.protocols import JobBackend __all__ = ["available", "get_backend"] @@ -42,6 +45,7 @@ # never need classes of their own. _BACKENDS: dict[str, str] = { "fake": "intercede._backends.fake:FakeBackend", # internal, for the test suites + "htcondor-ce": "intercede._backends.monolith.htcondorce:HTCondorCEBackend", } _TRANSPORTS: dict[str, str] = {} _SCHEDULERS: dict[str, str] = {} @@ -53,25 +57,39 @@ } -def get_backend(config: BackendConfig | Mapping[str, Any]) -> JobBackend: +def get_backend( + config: BackendConfig | Mapping[str, Any], credentials: CredentialProvider +) -> JobBackend: """Build the backend a configuration describes. - The backend is returned unopened: use it as ``async with registry.get_backend(config) as ce``. - Construction does no I/O, so building a backend for an unreachable resource succeeds. + The backend is returned unopened: use it as + ``async with registry.get_backend(config, credentials) as ce``. Construction + does no I/O and asks the provider for nothing, so building a backend for an + unreachable resource succeeds. The backend asks for credentials when it + first needs them, and again when they are about to expire. Args: config: A ``BackendConfig``, or the same data as a mapping, for example a parsed TOML table. + credentials: The consumer's provider. The consumer opens and closes it; + interCEde never does. One provider is normally shared by every + backend a task drives. Returns: The backend, not yet opened. Raises: - ConfigurationError: The configuration is invalid. + ConfigurationError: The configuration is invalid, or ``credentials`` + is not a ``CredentialProvider``. RegistryError: The configuration is valid, but this version of interCEde does not provide what it asks for. """ - return _build(validate_config(config)) + validated = validate_config(config) + if not isinstance(credentials, CredentialProvider): + raise ConfigurationError( + f"credentials must be a CredentialProvider, not a {type(credentials).__name__}" + ) + return _build(validated, credentials) def available() -> dict[str, tuple[str, ...]]: @@ -123,11 +141,14 @@ def _choices(annotation: Any) -> set[str]: return values -def _build(config: BackendConfig | FakeConfig) -> JobBackend: +def _build( + config: BackendConfig | FakeConfig, credentials: CredentialProvider +) -> JobBackend: """Build a backend from a validated configuration. Internal: the test suites call it directly. Args: config: The validated configuration of one backend, or the fake's settings. + credentials: The consumer's provider, for the parts that authenticate. Returns: The backend, not yet opened. A ``batch`` backend is composed of the @@ -139,14 +160,14 @@ def _build(config: BackendConfig | FakeConfig) -> JobBackend: what was built does not satisfy its protocol. """ if not isinstance(config, BatchLocalConfig | BatchSSHConfig): - built = _load("backend", config.kind).from_settings(config) + built = _load("backend", config.kind).from_settings(config, credentials) result: JobBackend = _checked("backend", config.kind, built, JobBackend) return result transport: Transport = _checked( "transport", config.transport, - _load("transport", config.transport).from_settings(config), + _load("transport", config.transport).from_settings(config, credentials), Transport, ) scheduler: Scheduler = _checked( diff --git a/tests/integration/README.md b/tests/integration/README.md index 2bbbadf..6b6a08a 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -5,7 +5,7 @@ called a *stack*. ```sh docker compose -f tests/integration/stacks//compose.yml up --wait -pixi run -e py313 pytest tests/integration -m "" --stack= --config=basic +pixi run -e py313 pytest tests/integration -n 4 -m "" --stack= --config=basic ``` Documentation: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6833ae8..5b77372 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,16 +1,61 @@ -"""Stack fixtures for the integration suite, selected with --stack and --config.""" +"""Stack fixtures for the integration suite, selected with --stack and --config. + +The tests marked ``contract`` are the shared suite: written once against the +interCEde protocols, run on every stack. A test that needs an optional +capability takes its fixture (``output_retriever``, ``killable``, +``purgeable``) and is skipped on a backend that does not have it. +""" from __future__ import annotations import os +import re import tomllib +from collections.abc import AsyncIterator, Callable from pathlib import Path -from typing import Any +from typing import Any, TypeVar +from urllib.parse import urlsplit import pytest +from intercede import registry +from intercede.auth.models import BearerToken, Credential +from intercede.auth.providers import StaticCredentials +from intercede.protocols import JobBackend, Killable, OutputRetriever, Purgeable from tests.integration.generate_matrix import STACKS_DIR, compose_env, load_manifest +# Nothing listens on port 1, so connecting is refused at once. +CLOSED_PORT = "127.0.0.1:1" + + +@pytest.hookimpl(tryfirst=True) # before `-m` deselects anything +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + """Select the shared suite for every stack, whatever its markers say. + + CI runs ``pytest -m ""`` with the stack's own expression. A shared + test gets the markers that expression asks for, so it is selected too. + """ + stack_id = config.getoption("--stack") + entries = {entry["id"]: entry for entry in load_manifest()} + if stack_id not in entries: + return # no stack: the `stack` fixture skips, or exits on an unknown id + for name in required_markers(entries[stack_id]["markers"]): + for item in items: + if item.get_closest_marker("contract") is not None: + item.add_marker(name) + + +def required_markers(expression: str) -> set[str]: + """Return the marker names a ``-m`` expression asks for, i.e. not negated.""" + words = re.findall(r"[\w-]+", expression) + return { + word + for previous, word in zip(["", *words], words) + if word not in ("and", "or", "not") and previous != "not" + } + @pytest.fixture(scope="session") def stack(request: pytest.FixtureRequest) -> dict[str, Any]: @@ -48,3 +93,122 @@ def client_config( pytest.skip(msg) with path.open("rb") as f: return tomllib.load(f) + + +# What a stack can declare its backend refuses, in [refusals] of its client config. +REFUSALS = frozenset({"containers", "url_inputs", "url_outputs"}) + + +@pytest.fixture(scope="session") +def refusals(client_config: dict[str, Any]) -> frozenset[str]: + """Return what this stack's backend cannot honour, as [refusals] declares it.""" + declared = client_config.get("refusals", {}) + if unknown := set(declared) - REFUSALS: + pytest.fail( + f"[refusals] has unknown keys {sorted(unknown)}; known: {sorted(REFUSALS)}" + ) + return frozenset(name for name, refused in declared.items() if refused) + + +def _token(folder: Path) -> Credential: + return BearerToken(value=(folder / "token").read_text().strip()) + + +def _tampered_token(credential: Credential) -> Credential: + # Same shape, wrong signature: what a revoked token, or one issued for + # another resource, looks like to the resource. + assert isinstance(credential, BearerToken) + value = credential.value.get_secret_value() + tail = "A" * 8 if value[-8:] != "A" * 8 else "B" * 8 + return BearerToken(value=value[:-8] + tail) + + +# How each credential type is read from the stack's credentials folder, and +# turned into one the resource rejects. +_READERS: dict[str, Callable[[Path], Credential]] = {"token": _token} +_TAMPERERS: dict[str, Callable[[Credential], Credential]] = {"token": _tampered_token} + + +@pytest.fixture +def credential(client_config: dict[str, Any], stack_dir: Path) -> Credential: + """Read the credential the stack minted at start, as [credentials] names it.""" + if "credentials" not in client_config: + pytest.skip("this stack's backend needs no credentials") + section = client_config["credentials"] + if section["type"] not in _READERS: + pytest.fail(f"credential type {section['type']!r} is not supported yet") + try: + # Read here and handed over as a value: the backend writes its own private copy. + return _READERS[section["type"]](stack_dir / section["path"]) + except FileNotFoundError as exc: + pytest.fail(f"{exc.filename} is missing: is the stack up?") + + +@pytest.fixture +def credentials(credential: Credential) -> StaticCredentials: + """Return a provider holding the stack's credential.""" + return StaticCredentials([credential]) + + +@pytest.fixture +def rejected_credentials( + client_config: dict[str, Any], credential: Credential +) -> StaticCredentials: + """Return a provider holding a credential of the right kind that the resource rejects.""" + return StaticCredentials( + [_TAMPERERS[client_config["credentials"]["type"]](credential)] + ) + + +@pytest.fixture +def unreachable_config(client_config: dict[str, Any]) -> dict[str, Any]: + """Return the stack's [backend], pointed at a closed port, as when the resource is down.""" + config = dict(client_config["backend"]) + if "endpoint" in config: + endpoint = urlsplit(config["endpoint"]) + if endpoint.scheme: # a URL, e.g. "https://ce:443/arex" + config["endpoint"] = endpoint._replace(netloc=CLOSED_PORT).geturl() + else: # "host:port" + config["endpoint"] = CLOSED_PORT + elif "host" in config: # reached over SSH + config["host"], port = CLOSED_PORT.split(":") + config["port"] = int(port) + else: + pytest.skip("this stack's backend reaches no resource over the network") + return config + + +@pytest.fixture +async def backend( + client_config: dict[str, Any], credentials: StaticCredentials +) -> AsyncIterator[JobBackend]: + """Yield the backend [backend] describes, built by the registry and open.""" + async with registry.get_backend(client_config["backend"], credentials) as opened: + yield opened + + +T = TypeVar("T") + + +def _capable(backend: JobBackend, capability: type[T]) -> T: + if not isinstance(backend, capability): + pytest.skip(f"the backend is not {capability.__name__}") + return backend + + +@pytest.fixture +def output_retriever(backend: JobBackend) -> OutputRetriever: + """Return the backend as an OutputRetriever; skip the test when it is not one.""" + return _capable(backend, OutputRetriever) + + +@pytest.fixture +def killable(backend: JobBackend) -> Killable: + """Return the backend as Killable; skip the test when it is not.""" + return _capable(backend, Killable) + + +@pytest.fixture +def purgeable(backend: JobBackend) -> Purgeable: + """Return the backend as Purgeable; skip the test when it is not.""" + return _capable(backend, Purgeable) diff --git a/tests/integration/generate_matrix.py b/tests/integration/generate_matrix.py index b812193..1e9b83d 100644 --- a/tests/integration/generate_matrix.py +++ b/tests/integration/generate_matrix.py @@ -19,6 +19,10 @@ STACKS_DIR = Path(__file__).parent / "stacks" REQUIRED_KEYS = ("id", "configs", "markers") +# pytest-xdist workers per job. The tests are independent and every backend +# queues concurrent jobs, so they run side by side; a stack whose backend +# cannot take concurrent jobs sets `workers: 1`. +DEFAULT_WORKERS = 4 def load_manifest() -> list[dict[str, Any]]: @@ -42,6 +46,16 @@ def is_ready(entry: dict[str, Any]) -> bool: ).is_dir() +def workers(entry: dict[str, Any]) -> int: + """Return how many pytest-xdist workers the stack's tests run with.""" + value = entry.get("workers", DEFAULT_WORKERS) + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise SystemExit( + f"stack {entry.get('id')!r}: `workers` must be a whole number >= 1, not {value!r}" + ) + return value + + def build_matrix(stacks: list[dict[str, Any]]) -> list[dict[str, Any]]: """Expand manifest entries into {stack x config x version} jobs.""" include = [] @@ -62,6 +76,7 @@ def build_matrix(stacks: list[dict[str, Any]]) -> list[dict[str, Any]]: "version": version, "markers": entry["markers"], "exec_in_container": entry.get("exec_in_container", False), + "workers": workers(entry), } ) return include diff --git a/tests/integration/jobs.py b/tests/integration/jobs.py new file mode 100644 index 0000000..c5c066d --- /dev/null +++ b/tests/integration/jobs.py @@ -0,0 +1,53 @@ +"""Helpers the shared suite uses to run jobs on a stack: payloads, waiting, cleanup.""" + +from __future__ import annotations + +import asyncio +import time +from collections.abc import Sequence +from pathlib import Path + +import pytest + +from intercede.models import JobHandle, JobStatus +from intercede.protocols import JobBackend, Purgeable + +# A job passes through the backend's own cycles (a CE's router, a scheduler's +# matchmaking) before it runs; a few minutes covers them on a slow CI runner. +FINISH_TIMEOUT_S = 300 +POLL_INTERVAL_S = 2 + +FINISHED = frozenset({JobStatus.DONE, JobStatus.FAILED, JobStatus.ABORTED}) +ACTIVE = frozenset({JobStatus.SUBMITTED, JobStatus.WAITING, JobStatus.RUNNING}) + + +def script(tmp_path: Path, body: str) -> str: + """Write a job script on this host and return its path. + + A local executable is shipped with the job by every backend, so the payload + does not depend on what is installed where it runs. + """ + path = tmp_path / "job.sh" + path.write_text(f"#!/bin/sh\n{body}\n") + path.chmod(0o755) + return str(path) + + +async def wait_for( + backend: JobBackend, handles: Sequence[JobHandle], states: frozenset[JobStatus] +) -> dict[JobHandle, JobStatus]: + """Poll until every job is in one of ``states``; fail after the timeout.""" + deadline = time.monotonic() + FINISH_TIMEOUT_S + while True: + statuses = dict(await backend.get_status(handles)) + if all(status in states for status in statuses.values()): + return statuses + if time.monotonic() > deadline: + pytest.fail(f"jobs did not reach {sorted(states)} in time: {statuses}") + await asyncio.sleep(POLL_INTERVAL_S) + + +async def release(backend: JobBackend, handles: Sequence[JobHandle]) -> None: + """Leave nothing behind on the stack, where the backend can release jobs.""" + if isinstance(backend, Purgeable) and handles: + await backend.purge(handles) diff --git a/tests/integration/stacks/_images/htcondor-ce/Dockerfile b/tests/integration/stacks/_images/htcondor-ce/Dockerfile new file mode 100644 index 0000000..2110478 --- /dev/null +++ b/tests/integration/stacks/_images/htcondor-ce/Dockerfile @@ -0,0 +1,52 @@ +# An HTCondor-CE in front of a one-machine HTCondor pool, for the `htcondor` stack. +# Rules: docs/dev/reference/integration-images.md +# +# The CE is the grid gateway interCEde submits to; its job router hands each job to the local +# pool, which runs it. Both run in this one container, started by entrypoint.sh (no systemd). +FROM almalinux:9 + +# Shared by every version, so above the ARGs: an ARG is part of the cache key of each later RUN. +# dnf leaves a mirror slower than minrate after `timeout` seconds, instead of finishing on it. +RUN printf '%s\n' 'fastestmirror=True' 'max_parallel_downloads=10' 'minrate=1M' 'timeout=15' \ + >> /etc/dnf/dnf.conf +# The metadata stays cached on purpose: the install below reuses it instead of downloading it again. +RUN set -eux; \ + dnf install -y epel-release dnf-plugins-core; \ + dnf config-manager --set-enabled crb + +# Leading edge, updated by Renovate (the `htcondor` group in renovate.json). +# renovate: datasource=github-tags depName=htcondor/htcondor +ARG HTCONDOR_VERSION=v25.14.1 +# renovate: datasource=github-tags depName=htcondor/htcondor-ce +ARG HTCONDOR_CE_VERSION=v25.7.0 + +# Long-term-support anchor, updated by hand: no `# renovate:` comment. +ARG HTCONDOR_LTS_VERSION=25.0.14 +ARG HTCONDOR_CE_LTS_VERSION=25.0.1 + +# Which of the two to build: `latest` or `lts`, from the stack's `versions` (compose passes it). +ARG BACKEND_VERSION=latest + +RUN set -eux; \ + case "$BACKEND_VERSION" in \ + latest) condor="${HTCONDOR_VERSION#v}"; ce="${HTCONDOR_CE_VERSION#v}"; series="${condor%%.*}.x" ;; \ + lts) condor="${HTCONDOR_LTS_VERSION}"; ce="${HTCONDOR_CE_LTS_VERSION}"; series="${condor%%.*}.0" ;; \ + *) echo "BACKEND_VERSION must be latest or lts, not '$BACKEND_VERSION'" >&2; exit 1 ;; \ + esac; \ + dnf install -y "https://research.cs.wisc.edu/htcondor/repo/${series}/htcondor-release-current.el9.noarch.rpm"; \ + dnf install -y "condor-${condor}" "htcondor-ce-${ce}" "htcondor-ce-condor-${ce}"; \ + dnf clean all; \ + rm -rf /var/cache/dnf + +# Jobs run as this unprivileged user: HTCondor refuses to run jobs as root. A token for +# submituser@users.htcondor.org (the CE's trust domain) maps to it. +RUN useradd --create-home submituser + +COPY pool.conf /etc/condor/config.d/90-intercede-pool.conf +COPY ce-router.conf /etc/condor-ce/config.d/90-intercede-router.conf +COPY entrypoint.sh /usr/local/bin/entrypoint.sh + +# The CE's single port (shared by all its daemons). +EXPOSE 9619 + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/tests/integration/stacks/_images/htcondor-ce/ce-router.conf b/tests/integration/stacks/_images/htcondor-ce/ce-router.conf new file mode 100644 index 0000000..6455af5 --- /dev/null +++ b/tests/integration/stacks/_images/htcondor-ce/ce-router.conf @@ -0,0 +1,6 @@ +# The pool behind the CE, in the same container (pool.conf). The packaged default names the +# schedd after the host, which the router cannot find (see SCHEDD_NAME in pool.conf). +JOB_ROUTER_SCHEDD2_NAME = pool@$(FULL_HOSTNAME) + +# Route new jobs every few seconds instead of every ten, so a test job reaches the pool quickly. +JOB_ROUTER_POLLING_PERIOD = 2 diff --git a/tests/integration/stacks/_images/htcondor-ce/entrypoint.sh b/tests/integration/stacks/_images/htcondor-ce/entrypoint.sh new file mode 100755 index 0000000..3aca6f9 --- /dev/null +++ b/tests/integration/stacks/_images/htcondor-ce/entrypoint.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Start the HTCondor pool and the HTCondor-CE in the foreground (no systemd), after writing a +# fresh client token to the shared credentials folder. Exits as soon as either daemon tree stops, +# so `docker compose` and the healthcheck see the failure. +set -euo pipefail + +CREDENTIALS="${CREDENTIALS_DIR:-/shared/credentials/htcondor}" +IDENTITY="${CLIENT_IDENTITY:-submituser@users.htcondor.org}" + +# The CE has no condor_ce_* wrapper for these two tools: run them with the CE's configuration. +ce() { CONDOR_CONFIG=/etc/condor-ce/condor_config "$@"; } + +# The CE signs ID tokens with its pool key. Create a new one at every start: credentials are +# throwaway and never part of the image (integration-images.md, rule 7). +install -d -m 0700 -o root -g root /etc/condor-ce/passwords.d +rm -f /etc/condor-ce/passwords.d/POOL +openssl rand -base64 48 | tr -d '\n' | ce condor_store_cred add -c -i - >/dev/null + +# The client's token, readable by everyone: the harness on the host copies it to a private file +# itself (integration-stacks.md, rule 6). +install -d -m 0755 "$CREDENTIALS" +ce condor_token_create -identity "$IDENTITY" > "$CREDENTIALS/token" +chmod 0644 "$CREDENTIALS/token" + +condor_master -f & # the pool +pool_pid=$! +/usr/share/condor-ce/condor_ce_startup -f & # the CE (checks its config, then runs its master) +ce_pid=$! + +# Daemon logs to stdout, so `docker compose logs` holds them all (integration-stacks.md, rule 4). +# HTCondor daemons can only log to files, and cannot open /dev/stdout as that file: they switch +# from root to the condor user, which may not reopen root's stdout. So the files are followed +# instead. `tail -F` waits for files that do not exist yet and prints a header per file. +follow() { + local tag="$1" dir="$2"; shift 2 + tail -n +1 -F "${@/#/$dir/}" 2>/dev/null | sed -u "s/^/[$tag] /" & +} +follow pool /var/log/condor MasterLog CollectorLog NegotiatorLog SchedLog StartLog \ + SharedPortLog ShadowLog StarterLog.slot1_1 StarterLog.slot1_2 StarterLog.slot1_3 StarterLog.slot1_4 +follow ce /var/log/condor-ce MasterLog CollectorLog SchedLog JobRouterLog SharedPortLog AuditLog + +trap 'kill "$pool_pid" "$ce_pid" 2>/dev/null; wait' TERM INT +wait -n "$pool_pid" "$ce_pid" +status=$? +echo "a daemon tree stopped (exit $status); stopping the container" >&2 +kill "$pool_pid" "$ce_pid" 2>/dev/null || true +exit "$status" diff --git a/tests/integration/stacks/_images/htcondor-ce/pool.conf b/tests/integration/stacks/_images/htcondor-ce/pool.conf new file mode 100644 index 0000000..805976d --- /dev/null +++ b/tests/integration/stacks/_images/htcondor-ce/pool.conf @@ -0,0 +1,13 @@ +# The batch pool behind the CE: one machine that is central manager, submit and execute node. +use ROLE: CentralManager +use ROLE: Submit +use ROLE: Execute +CONDOR_HOST = $(FULL_HOSTNAME) + +# The CE's job router finds this schedd by name (ce-router.conf). HTCondor resolves a plain +# name through DNS, and a container's reverse DNS gives another name than its hostname; a +# `name@host` name is taken as it is. +SCHEDD_NAME = pool@$(FULL_HOSTNAME) + +# Match jobs every few seconds instead of every minute, so a test job starts straight away. +NEGOTIATOR_INTERVAL = 2 diff --git a/tests/integration/stacks/_template/config/basic/intercede-client.toml b/tests/integration/stacks/_template/config/basic/intercede-client.toml index 785b352..2cdfc5e 100644 --- a/tests/integration/stacks/_template/config/basic/intercede-client.toml +++ b/tests/integration/stacks/_template/config/basic/intercede-client.toml @@ -4,10 +4,18 @@ id = "CHANGEME" # this stack's id in stacks.yml [backend] -kind = "CHANGEME" # arc | htcondor-ce | htcondor | ssh | local -endpoint = "CHANGEME" # e.g. "https://localhost:443/arex" or "localhost:2222" -queue = "CHANGEME" +# Exactly what a consumer passes to registry.get_backend. The fields of each kind are those of +# its configuration model (ARCConfig, HTCondorCEConfig, ...) in the API reference. +kind = "CHANGEME" # arc | htcondor-ce | cloud | batch (+ transport, scheduler) +endpoint = "CHANGEME" # e.g. "https://127.0.0.1:443/arex" or "127.0.0.1:9619" [credentials] -type = "CHANGEME" # token | proxy | ssh-key +type = "CHANGEME" # token | ssh-key path = "credentials/CHANGEME/" # under the ./credentials bind mount + +[refusals] +# What this backend cannot honour; the shared suite checks each is refused at submit. +# Leave out what it supports. +containers = true # running the executable in a container image +url_inputs = true # inputs the resource fetches from a URL itself +url_outputs = true # outputs the resource uploads to a URL itself diff --git a/tests/integration/stacks/htcondor/compose.yml b/tests/integration/stacks/htcondor/compose.yml new file mode 100644 index 0000000..f9bc43c --- /dev/null +++ b/tests/integration/stacks/htcondor/compose.yml @@ -0,0 +1,38 @@ +# The `htcondor` stack: an HTCondor-CE in front of a one-machine HTCondor pool. +# Rules: docs/dev/reference/integration-stacks.md +# The harness drives the service named `backend`: the CE, on 127.0.0.1:9619. +services: + backend: + # Until CI publishes images (#14): ghcr.io/diracgrid/intercede-testenv/htcondor-ce:. + # A local name per version, so `latest` and `lts` do not replace each other. + image: intercede-testenv/htcondor-ce:${BACKEND_VERSION:-latest} + build: + context: ../_images/htcondor-ce + args: + BACKEND_VERSION: ${BACKEND_VERSION:-latest} + hostname: backend + networks: + - stack + ports: + # Host-run harness: the CE advertises 127.0.0.1:9619 (config/basic/ce.conf), so the + # published port must be the same number. + - "127.0.0.1:9619:9619" + volumes: + # bind mount, not a named volume: the host-run harness must read the token + - ./credentials:/shared/credentials + # backend half of the configuration, next to its intercede-client.toml + - ./config/basic/ce.conf:/etc/condor-ce/config.d/99-basic.conf:ro + healthcheck: + # Ready when the CE's schedd is advertised and the pool has a slot to run jobs on. + test: + - CMD-SHELL + - >- + condor_ce_status -schedd -af Name | grep -q . && + condor_status -af Name | grep -q . + interval: 5s + timeout: 10s + retries: 24 + start_period: 30s + +networks: + stack: {} diff --git a/tests/integration/stacks/htcondor/config/basic/ce.conf b/tests/integration/stacks/htcondor/config/basic/ce.conf new file mode 100644 index 0000000..b565d12 --- /dev/null +++ b/tests/integration/stacks/htcondor/config/basic/ce.conf @@ -0,0 +1,19 @@ +# Backend half of the `basic` configuration: mounted into the CE as +# /etc/condor-ce/config.d/99-basic.conf. Changes together with intercede-client.toml +# (integration-stacks.md, rule 7). + +# Clients authenticate with an ID token (issue #23: the pure-IDTOKENS basic configuration). Out +# of the box the CE's collector accepts one, but its schedd takes only SCITOKENS and SSL from a +# client. FS stays for the CE's own daemons and tools inside the container. +SCHEDD.SEC_READ_AUTHENTICATION_METHODS = FS, TOKEN +SCHEDD.SEC_WRITE_AUTHENTICATION_METHODS = FS, TOKEN + +# The host reaches the CE through the published port 127.0.0.1:9619 (integration-stacks.md, +# rule 9). The CE's daemons all share that port, so advertising it works both from the host and +# from inside the container, where 127.0.0.1:9619 is the CE itself. +TCP_FORWARDING_HOST = 127.0.0.1 + +# The CE's schedd name, as clients pass it with `-name`. HTCondor tools look a plain name up in +# DNS and reject it when the host cannot resolve it (a container hostname, or `localhost`); a +# name of the form `name@host` is taken as it is. +SCHEDD_NAME = htcondor-ce@backend diff --git a/tests/integration/stacks/htcondor/config/basic/intercede-client.toml b/tests/integration/stacks/htcondor/config/basic/intercede-client.toml new file mode 100644 index 0000000..8026759 --- /dev/null +++ b/tests/integration/stacks/htcondor/config/basic/intercede-client.toml @@ -0,0 +1,22 @@ +# Client half of the `basic` configuration. See docs/dev/reference/integration-stacks.md +# Changes together with ce.conf, the backend half (rule 7). + +[stack] +id = "htcondor" # this stack's id in stacks.yml + +[backend] +kind = "htcondor-ce" +# The CE, published on the host (compose.yml) and advertised under this address (ce.conf). +endpoint = "127.0.0.1:9619" +# The CE's schedd name, as set in ce.conf. A `name@host` name: HTCondor tools reject a plain +# name the host cannot resolve, such as the container's hostname. +schedd = "htcondor-ce@backend" +# No `queue`: an HTCondor pool has no queues, and the CE routes every job to it. + +[credentials] +type = "token" # an ID token for submituser@users.htcondor.org (IDTOKENS) +path = "credentials/htcondor/" # holds `token`, minted at every container start + +[refusals] +# What this backend cannot honour; the shared suite checks it is refused at submit. +containers = true # no container_launcher configured above diff --git a/tests/integration/test_errors.py b/tests/integration/test_errors.py new file mode 100644 index 0000000..330477b --- /dev/null +++ b/tests/integration/test_errors.py @@ -0,0 +1,207 @@ +"""Shared suite: what a consumer sees when things go wrong, on every stack. + +Only failures that happen in practice: a credential the resource rejects or +that the provider does not hold, a resource that is down, a specification the +backend cannot honour, and per-job failures that must be reported, not raised. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from intercede import registry +from intercede.auth.providers import StaticCredentials +from intercede.exceptions import ( + AuthenticationError, + ResourceUnavailableError, + SpecificationRejectedError, +) +from intercede.models import ( + ContainerSpec, + CopyVars, + FileRef, + JobHandle, + JobStatus, + OutputMember, + OutputSpec, + SubmissionSpec, +) +from intercede.protocols import JobBackend, Killable, OutputRetriever, Purgeable +from tests.integration.jobs import release, script, wait_for + +pytestmark = pytest.mark.contract + + +async def submitted( + backend: JobBackend, tmp_path: Path, body: str = "true" +) -> JobHandle: + """Submit one job and return its handle.""" + submission = await backend.submit(SubmissionSpec(executable=script(tmp_path, body))) + return submission.handles[0] + + +# -------------------------------------------------------------------------- +# Whole-operation failures: raised, because no job could be dealt with +# -------------------------------------------------------------------------- + + +async def test_a_credential_the_resource_rejects( + backend: JobBackend, + client_config: dict[str, Any], + rejected_credentials: StaticCredentials, + tmp_path: Path, +): + handle = await submitted(backend, tmp_path) + try: + async with registry.get_backend( + client_config["backend"], rejected_credentials + ) as rejected: + with pytest.raises(AuthenticationError): + await rejected.submit( + SubmissionSpec(executable=script(tmp_path, "true")) + ) + with pytest.raises(AuthenticationError): + await rejected.get_status([handle]) + finally: + await release(backend, [handle]) + + +async def test_a_provider_without_the_credential( + client_config: dict[str, Any], credentials: StaticCredentials, tmp_path: Path +): + # `credentials` only makes the test skip on stacks that need none. + async with registry.get_backend( + client_config["backend"], StaticCredentials() + ) as empty: + with pytest.raises(AuthenticationError): + await empty.submit(SubmissionSpec(executable=script(tmp_path, "true"))) + + +async def test_a_resource_that_is_down( + backend: JobBackend, + unreachable_config: dict[str, Any], + credentials: StaticCredentials, + tmp_path: Path, +): + handle = await submitted(backend, tmp_path) + try: + async with registry.get_backend(unreachable_config, credentials) as down: + with pytest.raises(ResourceUnavailableError): + await down.submit(SubmissionSpec(executable=script(tmp_path, "true"))) + # A stored handle of a job on that resource, while it is down. + stored = handle.model_copy(update={"resource": down.resource}) + with pytest.raises(ResourceUnavailableError): + await down.get_status([stored]) + finally: + await release(backend, [handle]) + + +# -------------------------------------------------------------------------- +# Specifications refused before anything is submitted +# -------------------------------------------------------------------------- + + +async def test_a_secret_named_like_an_input(backend: JobBackend, tmp_path: Path): + (tmp_path / "pilot.cfg").write_text("settings") + spec = SubmissionSpec( + executable=script(tmp_path, "true"), + inputs=[FileRef(name="pilot.cfg", source=str(tmp_path / "pilot.cfg"))], + ) + with pytest.raises(SpecificationRejectedError, match="pilot.cfg"): + await backend.submit(spec, [CopyVars(secrets={"pilot.cfg": b"secret"})]) + + +async def test_a_local_input_that_does_not_exist(backend: JobBackend, tmp_path: Path): + spec = SubmissionSpec( + executable=script(tmp_path, "true"), + inputs=[FileRef(name="in.dat", source=str(tmp_path / "missing.dat"))], + ) + with pytest.raises(SpecificationRejectedError): + await backend.submit(spec) + + +async def test_an_output_left_for_collection_on_a_backend_that_cannot_collect( + backend: JobBackend, tmp_path: Path +): + if isinstance(backend, OutputRetriever): + pytest.skip("the backend collects outputs") + spec = SubmissionSpec( + executable=script(tmp_path, "true"), + outputs=OutputSpec(members=[OutputMember(pattern="result.txt")]), + ) + with pytest.raises(SpecificationRejectedError): + await backend.submit(spec) + + +# A request per refusal a stack can declare in [refusals] of its client config. +REFUSED_SPECS = { + "containers": {"container": ContainerSpec(image="docker://almalinux:9")}, + "url_inputs": { + "inputs": [FileRef(name="data.txt", source="https://example.org/data.txt")] + }, + "url_outputs": { + "outputs": OutputSpec( + members=[ + OutputMember( + pattern="result.txt", + destination="https://storage.example.org/result.txt", + ) + ] + ) + }, +} + + +@pytest.mark.parametrize("refusal", sorted(REFUSED_SPECS)) +async def test_what_the_stack_cannot_honour_is_refused_at_submit( + backend: JobBackend, refusals: frozenset[str], refusal: str, tmp_path: Path +): + """Refused at submit, not accepted and then failed at the resource.""" + if refusal not in refusals: + pytest.skip(f"this stack's backend supports {refusal} (see [refusals])") + spec = SubmissionSpec(executable=script(tmp_path, "true"), **REFUSED_SPECS[refusal]) + with pytest.raises(SpecificationRejectedError): + await backend.submit(spec) + + +# -------------------------------------------------------------------------- +# Per-job failures: reported in the result, never raised +# -------------------------------------------------------------------------- + + +async def test_the_output_of_a_running_job( + backend: JobBackend, output_retriever: OutputRetriever, tmp_path: Path +): + handle = await submitted(backend, tmp_path, "sleep 600") + try: + await wait_for(backend, [handle], frozenset({JobStatus.RUNNING})) + + outcome = (await output_retriever.get_output([handle], tmp_path / "out"))[ + handle + ] + + assert not outcome.ok and outcome.reason + finally: + await release(backend, [handle]) + + +async def test_a_job_of_another_resource(backend: JobBackend, tmp_path: Path): + """A handle the consumer stored for a different resource is not this backend's job.""" + handle = await submitted(backend, tmp_path) + try: + theirs = handle.model_copy(update={"resource": "elsewhere.example.org:9619"}) + assert (await backend.get_status([theirs]))[theirs] == JobStatus.UNKNOWN + outcomes = [] + if isinstance(backend, OutputRetriever): + outcomes.append(await backend.get_output([theirs], tmp_path / "out")) + if isinstance(backend, Killable): + outcomes.append(await backend.kill([theirs])) + if isinstance(backend, Purgeable): + outcomes.append(await backend.purge([theirs])) + for outcome in outcomes: + assert not outcome[theirs].ok and outcome[theirs].reason + finally: + await release(backend, [handle]) diff --git a/tests/integration/test_kill.py b/tests/integration/test_kill.py new file mode 100644 index 0000000..11674da --- /dev/null +++ b/tests/integration/test_kill.py @@ -0,0 +1,31 @@ +"""Shared suite: Killable, on every stack whose backend has it.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from intercede.models import JobStatus, SubmissionSpec +from intercede.protocols import JobBackend, Killable +from tests.integration.jobs import release, script, wait_for + +pytestmark = pytest.mark.contract + + +async def test_kill_stops_a_running_job( + backend: JobBackend, killable: Killable, tmp_path: Path +): + submission = await backend.submit( + SubmissionSpec(executable=script(tmp_path, "sleep 600")) + ) + handles = list(submission.handles.values()) + try: + await wait_for(backend, handles, frozenset({JobStatus.RUNNING})) + + outcomes = await killable.kill(handles) + + assert outcomes[handles[0]].ok, outcomes[handles[0]].reason + await wait_for(backend, handles, frozenset({JobStatus.ABORTED})) + finally: + await release(backend, handles) diff --git a/tests/integration/test_manifest.py b/tests/integration/test_manifest.py index 8304ae5..e591a1e 100644 --- a/tests/integration/test_manifest.py +++ b/tests/integration/test_manifest.py @@ -2,7 +2,15 @@ from __future__ import annotations -from tests.integration.generate_matrix import STACKS_DIR, compose_env, load_manifest +from _pytest.mark.expression import Expression + +from tests.integration.conftest import required_markers +from tests.integration.generate_matrix import ( + STACKS_DIR, + compose_env, + load_manifest, + workers, +) def test_entries_have_required_keys(): @@ -12,6 +20,11 @@ def test_entries_have_required_keys(): assert entry.get("markers"), f"stack {entry['id']}: `markers` is required" +def test_workers_are_valid(): + for entry in load_manifest(): + workers(entry) # raises SystemExit on a bad value + + def test_stack_ids_are_unique(): ids = [entry["id"] for entry in load_manifest()] assert len(ids) == len(set(ids)), f"duplicate stack ids in stacks.yml: {ids}" @@ -29,3 +42,13 @@ def test_landed_stacks_follow_the_layout(): for config in entry["configs"]: client_toml = own_dir / "config" / config / "intercede-client.toml" assert client_toml.is_file(), f"{client_toml} is missing" + + +def test_every_stack_selects_the_shared_suite(): + """The markers a shared test gets from a stack must satisfy that stack's `-m`.""" + for entry in load_manifest(): + given = required_markers(entry["markers"]) + selected = Expression.compile(entry["markers"]).evaluate( + lambda name, given=given, **_: name in given + ) + assert selected, f"stack {entry['id']}: its markers would skip the shared suite" diff --git a/tests/integration/test_output.py b/tests/integration/test_output.py new file mode 100644 index 0000000..5626c32 --- /dev/null +++ b/tests/integration/test_output.py @@ -0,0 +1,155 @@ +"""Shared suite: OutputRetriever, on every stack whose backend has it.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from intercede.models import ( + CopyVars, + FileRef, + JobStatus, + OutputMember, + OutputSpec, + SubmissionSpec, +) +from intercede.protocols import JobBackend, OutputRetriever +from tests.integration.jobs import release, script, wait_for + +pytestmark = pytest.mark.contract + +DONE = frozenset({JobStatus.DONE}) + + +async def test_inputs_environment_and_outputs_round_trip( + backend: JobBackend, output_retriever: OutputRetriever, tmp_path: Path +): + (tmp_path / "greeting.txt").write_text("hello") + spec = SubmissionSpec( + executable=script( + tmp_path, + 'cat greeting.txt; echo " $WHO"; echo to-stderr >&2; echo "$WHO" > result.txt', + ), + inputs=[FileRef(name="greeting.txt", source=str(tmp_path / "greeting.txt"))], + outputs=OutputSpec(members=[OutputMember(pattern="result.txt")]), + ) + people = ("ann", "bob") + submission = await backend.submit( + spec, copies=[CopyVars(environment={"WHO": who}) for who in people] + ) + handles = [submission.handles[i] for i in range(len(people))] + try: + await wait_for(backend, handles, DONE) + + outputs = await output_retriever.get_output(handles, tmp_path / "out") + + for handle, who in zip(handles, people): + outcome = outputs[handle] + assert outcome.ok, outcome.reason + output = outcome.value + assert output is not None + assert output.stdout is not None + assert output.stdout.read_text() == f"hello {who}\n" + assert output.stderr is not None + assert output.stderr.read_text() == "to-stderr\n" + assert output.files["result.txt"].read_text() == f"{who}\n" + finally: + await release(backend, handles) + + +async def test_output_can_be_collected_again( + backend: JobBackend, output_retriever: OutputRetriever, tmp_path: Path +): + """Retrieval is repeatable until purge: a lost download is not lost output.""" + submission = await backend.submit( + SubmissionSpec(executable=script(tmp_path, "echo once")) + ) + handles = list(submission.handles.values()) + try: + await wait_for(backend, handles, DONE) + + first = await output_retriever.get_output(handles, tmp_path / "first") + second = await output_retriever.get_output(handles, tmp_path / "second") + + for outputs in (first, second): + outcome = outputs[handles[0]] + assert outcome.ok, outcome.reason + assert outcome.value is not None + assert outcome.value.stdout is not None + assert outcome.value.stdout.read_text() == "once\n" + finally: + await release(backend, handles) + + +async def test_each_copy_gets_its_own_identity_and_secret( + backend: JobBackend, output_retriever: OutputRetriever, tmp_path: Path +): + """The failure that matters most, because the jobs still run: every copy alike.""" + spec = SubmissionSpec( + executable=script( + tmp_path, + 'echo "$WHO"; cat pilot.secret; echo; stat -c %a pilot.secret', + ) + ) + copies = [ + CopyVars( + environment={"WHO": f"copy-{i}"}, secrets={"pilot.secret": b"secret-%d" % i} + ) + for i in range(3) + ] + submission = await backend.submit(spec, copies) + handles = [submission.handles[i] for i in range(len(copies))] + try: + await wait_for(backend, handles, DONE) + + outputs = await output_retriever.get_output(handles, tmp_path / "out") + + for i, handle in enumerate(handles): + output = outputs[handle].value + assert output is not None and output.stdout is not None, outputs[ + handle + ].reason + assert output.stdout.read_text().splitlines() == [ + f"copy-{i}", + f"secret-{i}", + "600", # readable by the job's user only + ] + finally: + await release(backend, handles) + + +async def test_stdout_and_stderr_stay_apart_from_payload_files( + backend: JobBackend, output_retriever: OutputRetriever, tmp_path: Path +): + """A payload may write files named like a backend's own streams (ARC calls them so).""" + spec = SubmissionSpec( + executable=script( + tmp_path, + "echo to-stdout; echo to-stderr >&2; " + "echo file-stdout > stdout; echo file-stderr > stderr; echo 42 > result.txt", + ), + outputs=OutputSpec(include_unlisted=True), # bring back every file + ) + submission = await backend.submit(spec) + handles = list(submission.handles.values()) + try: + await wait_for(backend, handles, DONE) + + outcome = (await output_retriever.get_output(handles, tmp_path / "out"))[ + handles[0] + ] + + assert outcome.ok, outcome.reason + output = outcome.value + assert output is not None + assert output.stdout is not None and output.stdout.read_text() == "to-stdout\n" + assert output.stderr is not None and output.stderr.read_text() == "to-stderr\n" + assert output.files["stdout"].read_text() == "file-stdout\n" + assert output.files["stderr"].read_text() == "file-stderr\n" + assert output.files["result.txt"].read_text() == "42\n" + # The streams and the log are never payload files. + streams = {output.stdout, output.stderr, output.log} + assert not streams & set(output.files.values()) + finally: + await release(backend, handles) diff --git a/tests/integration/test_purge.py b/tests/integration/test_purge.py new file mode 100644 index 0000000..f10f27d --- /dev/null +++ b/tests/integration/test_purge.py @@ -0,0 +1,32 @@ +"""Shared suite: Purgeable, on every stack whose backend has it.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from intercede.models import JobStatus, SubmissionSpec +from intercede.protocols import JobBackend, OutputRetriever, Purgeable +from tests.integration.jobs import ACTIVE, script, wait_for + +pytestmark = pytest.mark.contract + + +async def test_purge_releases_a_finished_job( + backend: JobBackend, purgeable: Purgeable, tmp_path: Path +): + submission = await backend.submit( + SubmissionSpec(executable=script(tmp_path, "true")) + ) + handles = list(submission.handles.values()) + await wait_for(backend, handles, frozenset({JobStatus.DONE})) + + outcomes = await purgeable.purge(handles) + + assert outcomes[handles[0]].ok, outcomes[handles[0]].reason + statuses = await backend.get_status(handles) + assert statuses[handles[0]] not in ACTIVE + if isinstance(backend, OutputRetriever): + output = await backend.get_output(handles, tmp_path / "out") + assert not output[handles[0]].ok diff --git a/tests/integration/test_status.py b/tests/integration/test_status.py new file mode 100644 index 0000000..a070598 --- /dev/null +++ b/tests/integration/test_status.py @@ -0,0 +1,53 @@ +"""Shared suite: StatusReporter, on every stack.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from intercede.models import JobID, JobStatus, SubmissionSpec +from intercede.protocols import JobBackend +from tests.integration.jobs import FINISHED, release, script, wait_for + +pytestmark = pytest.mark.contract + + +async def test_a_job_that_succeeds_is_done(backend: JobBackend, tmp_path: Path): + submission = await backend.submit( + SubmissionSpec(executable=script(tmp_path, "true")) + ) + handles = list(submission.handles.values()) + try: + assert await wait_for(backend, handles, FINISHED) == { + handles[0]: JobStatus.DONE + } + finally: + await release(backend, handles) + + +async def test_a_job_that_exits_non_zero_failed(backend: JobBackend, tmp_path: Path): + submission = await backend.submit( + SubmissionSpec(executable=script(tmp_path, "exit 3")) + ) + handles = list(submission.handles.values()) + try: + assert await wait_for(backend, handles, FINISHED) == { + handles[0]: JobStatus.FAILED + } + finally: + await release(backend, handles) + + +async def test_an_unknown_job_is_unknown(backend: JobBackend, tmp_path: Path): + submission = await backend.submit( + SubmissionSpec(executable=script(tmp_path, "true")) + ) + handles = list(submission.handles.values()) + try: + unknown = handles[0].model_copy(update={"id": JobID("999999.0")}) + statuses = await backend.get_status([handles[0], unknown]) + assert set(statuses) == {handles[0], unknown} + assert statuses[unknown] == JobStatus.UNKNOWN + finally: + await release(backend, handles) diff --git a/tests/integration/test_submit.py b/tests/integration/test_submit.py new file mode 100644 index 0000000..7b73b51 --- /dev/null +++ b/tests/integration/test_submit.py @@ -0,0 +1,45 @@ +"""Shared suite: Submitter, on every stack.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from intercede.models import JobHandle, JobStatus, SubmissionSpec +from intercede.protocols import JobBackend +from tests.integration.jobs import release, script + +pytestmark = pytest.mark.contract + + +async def test_submit_returns_one_handle_per_copy( + backend: JobBackend, client_config: dict[str, Any], tmp_path: Path +): + submission = await backend.submit( + SubmissionSpec(executable=script(tmp_path, "true")), copies=2 + ) + handles = list(submission.handles.values()) + try: + assert submission.failures == {} + assert sorted(submission.handles) == [0, 1] + assert len(set(handles)) == 2 + for handle in handles: + assert handle.backend == client_config["backend"]["kind"] + assert handle.resource == backend.resource + finally: + await release(backend, handles) + + +async def test_a_stored_handle_finds_the_job(backend: JobBackend, tmp_path: Path): + submission = await backend.submit( + SubmissionSpec(executable=script(tmp_path, "true")) + ) + handles = list(submission.handles.values()) + try: + stored = JobHandle.model_validate_json(handles[0].model_dump_json()) + statuses = await backend.get_status([stored]) + assert statuses[stored] != JobStatus.UNKNOWN + finally: + await release(backend, handles) diff --git a/tests/unit/_registry_dummies.py b/tests/unit/_registry_dummies.py index 5dc02fe..345b4eb 100644 --- a/tests/unit/_registry_dummies.py +++ b/tests/unit/_registry_dummies.py @@ -19,10 +19,13 @@ async def __aexit__(self, *exc_info: object) -> None: ... class DummyTransport(_Lifecycle): def __init__(self, settings: Any) -> None: self.settings = settings + self.credentials: Any = None @classmethod - def from_settings(cls, settings: Any) -> DummyTransport: - return cls(settings) + def from_settings(cls, settings: Any, credentials: Any) -> DummyTransport: + built = cls(settings) + built.credentials = credentials + return built async def run(self, argv: Any) -> Any: ... @@ -74,10 +77,13 @@ class DummyBackend(_Lifecycle): def __init__(self, settings: Any) -> None: self.settings = settings + self.credentials: Any = None @classmethod - def from_settings(cls, settings: Any) -> DummyBackend: - return cls(settings) + def from_settings(cls, settings: Any, credentials: Any) -> DummyBackend: + built = cls(settings) + built.credentials = credentials + return built async def submit(self, spec: Any, copies: Any = 1) -> Any: ... diff --git a/tests/unit/backends/__init__.py b/tests/unit/backends/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/backends/_htcondor/__init__.py b/tests/unit/backends/_htcondor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/backends/_htcondor/test_core.py b/tests/unit/backends/_htcondor/test_core.py new file mode 100644 index 0000000..185d2e6 --- /dev/null +++ b/tests/unit/backends/_htcondor/test_core.py @@ -0,0 +1,360 @@ +"""The shared HTCondor knowledge: submit descriptions, commands and the state mapping. + +Expected values were checked against HTCondor 25 (condor_submit -dry-run and a +live schedd): what is asserted here is what HTCondor actually does with them. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from intercede import JobStatus +from intercede._backends._htcondor import commands, status, submit +from intercede._backends._htcondor.submit import CopyPlan, SubmitPlan, render + +# -------------------------------------------------------------------------- +# Quoting +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("arguments", "quoted"), + [ + (["plain"], "\"'plain'\""), + (["two words"], "\"'two words'\""), + (["it's"], "\"'it''s'\""), + (['say "hi"'], '"\'say ""hi""\'"'), + (["cost $(HOME) $5"], "\"'cost $(DOLLAR)(HOME) $(DOLLAR)5'\""), + ([""], "\"''\""), + (["a", "b"], "\"'a' 'b'\""), + ], + ids=[ + "plain", + "space", + "single-quote", + "double-quote", + "dollar", + "empty", + "several", + ], +) +def test_quote_arguments(arguments: list[str], quoted: str): + assert submit.quote_arguments(arguments) == quoted + + +def test_quote_environment_sorts_and_quotes(): + quoted = submit.quote_environment({"STAMP": "a b", "X": "$(Y)", "A": "it's"}) + assert quoted == "\"A='it''s' STAMP='a b' X='$(DOLLAR)(Y)'\"" + + +@pytest.mark.parametrize("name", ["", "A=B", "WITH SPACE", "DASH-ED", "$X"]) +def test_environment_names_must_be_identifiers(name: str): + with pytest.raises(ValueError, match="not a valid environment variable name"): + submit.quote_environment({name: "x"}) + + +@pytest.mark.parametrize("value", ["line\nbreak", "carriage\rreturn"]) +def test_line_breaks_are_refused(value: str): + with pytest.raises(ValueError, match="line break"): + submit.quote_arguments([value]) + with pytest.raises(ValueError, match="line break"): + submit.quote_environment({"X": value}) + + +# -------------------------------------------------------------------------- +# Submit descriptions +# -------------------------------------------------------------------------- + + +def plan(**changes: Any) -> SubmitPlan: + values: dict[str, Any] = { + "executable": "/work/payload.sh", + "transfer_executable": True, + "arguments": ["a"], + "copies": [ + CopyPlan("/work/0", {"STAMP": "s0"}, ["secrets/"]), + CopyPlan("/work/1", {"STAMP": "s1"}), + ], + } + return SubmitPlan(**{**values, **changes}) + + +def lines(text: str) -> list[str]: + return text.splitlines() + + +class TestRender: + def test_one_process_per_copy(self): + assert lines(render(plan()))[-1] == "queue 2" + + def test_per_copy_values_are_indexed_by_proc_id(self): + text = render(plan()) + assert "ic_dir_0 = /work/0" in lines(text) + assert "ic_dir_1 = /work/1" in lines(text) + assert "ic_env_0 = \"STAMP='s0'\"" in lines(text) + assert "ic_in_0 = secrets/" in lines(text) + assert "ic_in_1 = " in lines(text) + assert "initialdir = $(ic_dir_$(ProcId))" in lines(text) + assert "environment = $(ic_env_$(ProcId))" in lines(text) + + def test_inputs_shared_and_per_copy(self): + text = render(plan(inputs=["/work/inputs/", "https://example.org/in.dat"])) + assert ( + "transfer_input_files = /work/inputs/, https://example.org/in.dat, $(ic_in_$(ProcId))" + in lines(text) + ) + + @pytest.mark.parametrize( + ("outputs", "expected"), + [ + ( + [], + 'transfer_output_files = ""', + ), # nothing but stdout, stderr and the log + (["result.txt", "dir/x"], "transfer_output_files = result.txt, dir/x"), + (None, None), # every file the payload creates + ], + ids=["none", "listed", "all"], + ) + def test_outputs(self, outputs: list[str] | None, expected: str | None): + text = render(plan(outputs=outputs)) + found = [ + line for line in lines(text) if line.startswith("transfer_output_files") + ] + assert found == ([expected] if expected else []) + + def test_output_remaps_upload_to_urls(self): + text = render( + plan( + outputs=["a", "b"], + output_remaps={"b": "https://s/b", "a": "https://s/a"}, + ) + ) + assert 'transfer_output_remaps = "a = https://s/a; b = https://s/b"' in lines( + text + ) + + def test_resources_for_a_schedd_and_for_a_ce_router(self): + text = render(plan(cpus=8, memory_mb=4000, wall_time_s=7200, queue="long")) + for line in [ + "request_cpus = 8", + "+xcount = 8", + "request_memory = 4000", + "+maxMemory = 4000", + "+BatchRuntime = 7200", + '+queue = "long"', + ]: + assert line in lines(text) + + def test_finished_jobs_stay_until_purged(self): + # The one lifetime rule interCEde sets: output can be collected + # again until the consumer releases the job. + assert "leave_in_queue = (JobStatus == 4)" in lines(render(plan())) + + @pytest.mark.parametrize( + "command", + [ + "on_exit_hold", + "on_exit_remove", + "periodic_hold", + "periodic_release", + "periodic_remove", + "max_retries", + "requirements", + ], + ) + def test_sets_no_policy_on_the_job(self, command: str): + # Holding, releasing, removing or retrying jobs is the consumer's + # policy, never interCEde's. + assert not any(line.startswith(command) for line in lines(render(plan()))) + + def test_macros_in_paths_are_escaped(self): + assert "ic_dir_0 = /work/$(DOLLAR)0" in lines( + render(plan(copies=[CopyPlan("/work/$0")])) + ) + + @pytest.mark.parametrize("field", ["inputs", "outputs"]) + def test_file_lists_cannot_hold_a_comma(self, field: str): + with pytest.raises(ValueError, match="comma"): + render(plan(**{field: ["a,b"]})) + + def test_paths_cannot_hold_a_line_break(self): + with pytest.raises(ValueError, match="line break"): + render(plan(copies=[CopyPlan("/work/a\nqueue 100")])) + + +# -------------------------------------------------------------------------- +# Commands +# -------------------------------------------------------------------------- + + +class TestCommands: + def test_submit_to_the_local_schedd_spools(self): + assert commands.submit("/w/job.sub") == [ + "condor_submit", + "-terse", + "-spool", + "/w/job.sub", + ] + + @pytest.mark.parametrize( + ("count", "sizes"), + [(100, [100]), (101, [100, 1]), (250, [100, 100, 50])], + ) + def test_chunks(self, count: int, sizes: list[int]): + ids = [status.JobID(f"1.{i}") for i in range(count)] + chunked = commands.chunks(ids) + assert [len(chunk) for chunk in chunked] == sizes + assert [i for chunk in chunked for i in chunk] == ids + + +class TestParseSubmit: + @pytest.mark.parametrize( + ("raw", "ids"), + [ + ("1.0 - 1.1\n", ["1.0", "1.1"]), + (" 42.0 - 42.0 ", ["42.0"]), + ("\n7.3 - 7.5\n\n", ["7.3", "7.4", "7.5"]), + ], + ) + def test_terse_output(self, raw: str, ids: list[str]): + assert commands.parse_submit(raw) == ids + + @pytest.mark.parametrize( + "raw", + [ + "", + "Submitting job(s).\n1 job(s) submitted to cluster 1.", + "1.0 - 2.0", + "1.3 - 1.1", + "1.0 - 1.1\n2.0 - 2.1", + "x", + ], + ids=["empty", "verbose", "two-clusters", "backwards", "two-lines", "garbage"], + ) + def test_anything_else_is_refused(self, raw: str): + with pytest.raises(ValueError, match="unexpected condor_submit output"): + commands.parse_submit(raw) + + +def test_parse_remove(): + raw = "Job 1.0 marked for removal\nJob 1.2 marked for removal\nCouldn't find/remove all jobs matching 1.1\n" + assert commands.parse_remove(raw) == {"1.0", "1.2"} + assert commands.parse_remove("") == set() + + +# -------------------------------------------------------------------------- +# State mapping +# -------------------------------------------------------------------------- + + +def ad(**fields: Any) -> dict[str, Any]: + return {"ClusterId": 3, "ProcId": 1, **fields} + + +class TestJobStatus: + @pytest.mark.parametrize( + ("code", "expected"), + [ + (1, JobStatus.WAITING), + (2, JobStatus.RUNNING), + (4, JobStatus.DONE), + (6, JobStatus.RUNNING), + (7, JobStatus.WAITING), + ], + ids=["idle", "running", "completed", "transferring-output", "suspended"], + ) + def test_plain_states(self, code: int, expected: JobStatus): + assert status.job_status(ad(JobStatus=code)) == expected + + @pytest.mark.parametrize( + "code", + [8, None], + ids=["unexpected", "missing"], + ) + def test_unknown_codes(self, code: Any): + assert status.job_status(ad(JobStatus=code)) == JobStatus.UNKNOWN + + @pytest.mark.parametrize( + ("fields", "expected"), + [ + ({"ExitCode": 0, "ExitBySignal": False}, JobStatus.DONE), + ({"ExitCode": 3, "ExitBySignal": False}, JobStatus.FAILED), + ({"ExitBySignal": True, "ExitSignal": 9}, JobStatus.FAILED), + ({}, JobStatus.DONE), + ], + ids=["exit-0", "exit-3", "signal", "no-exit-info"], + ) + def test_finished_payload_result_comes_from_its_exit( + self, fields: dict[str, Any], expected: JobStatus + ): + # Read from what HTCondor records; no hold is put on the job to tell. + assert status.job_status(ad(JobStatus=4, **fields)) == expected + + def test_spooling_is_waiting(self): + assert ( + status.job_status(ad(JobStatus=5, HoldReasonCode=16)) == JobStatus.WAITING + ) + + @pytest.mark.parametrize( + "fields", + [{"HoldReasonCode": 1}, {"HoldReasonCode": 3, "HoldReasonSubCode": 55}, {}], + ids=["by-user", "by-site-policy", "no-reason"], + ) + def test_any_other_hold_is_aborted(self, fields: dict[str, Any]): + assert status.job_status(ad(JobStatus=5, **fields)) == JobStatus.ABORTED + + @pytest.mark.parametrize( + ("fields", "expected"), + [ + ({"LastJobStatus": 4, "ExitCode": 0}, JobStatus.DONE), + ({"LastJobStatus": 4}, JobStatus.DONE), + ({"LastJobStatus": 4, "ExitCode": 2}, JobStatus.FAILED), + ({"LastJobStatus": 4, "ExitBySignal": True}, JobStatus.FAILED), + ({"LastJobStatus": 5, "ExitCode": 3}, JobStatus.ABORTED), + ({"LastJobStatus": 2}, JobStatus.ABORTED), + ({"LastJobStatus": 1}, JobStatus.ABORTED), + ({}, JobStatus.ABORTED), + ], + ids=[ + "purged-done", + "purged-done-no-code", + "purged-failed", + "purged-signal", + "removed-while-held", + "killed-running", + "killed-idle", + "bare", + ], + ) + def test_removed_jobs_keep_their_result( + self, fields: dict[str, Any], expected: JobStatus + ): + assert status.job_status(ad(JobStatus=3, **fields)) == expected + + +class TestParseAds: + def test_condor_q_prints_nothing_for_no_match(self): + assert status.parse_ads("") == [] + assert status.parse_ads("\n \n") == [] + + def test_condor_history_prints_an_empty_list(self): + assert status.parse_ads("[\n]\n") == [] + + @pytest.mark.parametrize("raw", ["{}", "[1, 2]", '"text"', "not json"]) + def test_anything_else_is_refused(self, raw: str): + with pytest.raises(ValueError): + status.parse_ads(raw) + + def test_every_status_query_asks_for_what_the_mapping_reads(self): + assert { + "ClusterId", + "ProcId", + "JobStatus", + "LastJobStatus", + "ExitCode", + "ExitBySignal", + "HoldReasonCode", + } <= set(status.STATE_ATTRIBUTES) diff --git a/tests/unit/backends/fake/__init__.py b/tests/unit/backends/fake/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_fake_backend.py b/tests/unit/backends/fake/test_fake_backend.py similarity index 100% rename from tests/unit/test_fake_backend.py rename to tests/unit/backends/fake/test_fake_backend.py diff --git a/tests/unit/backends/htcondorce/__init__.py b/tests/unit/backends/htcondorce/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/backends/htcondorce/_fake_condor.py b/tests/unit/backends/htcondorce/_fake_condor.py new file mode 100644 index 0000000..bab99d7 --- /dev/null +++ b/tests/unit/backends/htcondorce/_fake_condor.py @@ -0,0 +1,204 @@ +"""A simulated set of HTCondor tools, and helpers for driving the backend with it. + +``FakeCondor`` answers the way HTCondor 25 does (checked against a live schedd): +``condor_q`` prints nothing for unknown jobs, ``condor_history`` an empty list, +``condor_rm`` exits 1 for jobs it cannot find, and ``condor_transfer_data`` +writes into each job's ``SUBMIT_Iwd``. +""" + +from __future__ import annotations + +import json +import re +import stat +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from intercede import HTCondorCEConfig, JobHandle, JobID, SubmissionSpec +from intercede._backends.batch.protocols import CommandResult +from intercede._backends.monolith.htcondorce import HTCondorCEBackend +from intercede.auth import BearerToken, StaticCredentials + +ENDPOINT = "ce.example.org:9619" +SPEC = SubmissionSpec(executable="/bin/true") +TOKEN = "TOKEN-VALUE" # noqa: S105 (a fake token the tests look for) + + +@dataclass +class Call: + argv: list[str] + env: dict[str, str] + files: dict[str, tuple[int, bytes]] = field( + default_factory=dict + ) # path -> (mode, content) + + +class FakeCondor: + """Plays condor_submit, condor_q, condor_history, condor_rm and condor_transfer_data.""" + + def __init__(self) -> None: + self.calls: list[Call] = [] + self.queue: dict[str, dict[str, Any]] = {} + self.history: dict[str, dict[str, Any]] = {} + self.outputs: dict[str, dict[str, bytes]] = {} # files a job "created" + self.descriptions: list[str] = [] + self.next_cluster = 1 + self.fail: dict[str, CommandResult] = {} # tool -> forced result + + async def __call__( + self, argv: Sequence[str], env: Mapping[str, str], timeout: float + ) -> CommandResult: + call = Call(list(argv), dict(env)) + self.calls.append(call) + self._snapshot(call) + tool, args = argv[0], [a for a in argv[1:] if a != "-debug"] + if tool in self.fail: + return self.fail[tool] + ids = [a for a in args if re.fullmatch(r"\d+\.\d+", a)] + if tool == "condor_submit": + return self._submit(Path(args[-1])) + if tool == "condor_q": + return self._ads(self.queue, ids, args, empty="") + if tool == "condor_history": + return self._ads(self.history, ids, args, empty="[\n]\n") + if tool == "condor_rm": + return self._remove(ids) + if tool == "condor_transfer_data": + return self._transfer(ids) + raise AssertionError(f"unexpected tool {tool}") + + # -- the tools ----------------------------------------------------------- + + def _submit(self, description: Path) -> CommandResult: + text = description.read_text() + self.descriptions.append(text) + count = int(re.search(r"^queue (\d+)$", text, re.M).group(1)) # type: ignore[union-attr] + cluster = self.next_cluster + self.next_cluster += 1 + for proc in range(count): + iwd = re.search(rf"^ic_dir_{proc} = (.*)$", text, re.M).group(1) # type: ignore[union-attr] + out = re.search(r"^output = (.*)$", text, re.M).group(1) # type: ignore[union-attr] + err = re.search(r"^error = (.*)$", text, re.M).group(1) # type: ignore[union-attr] + self.queue[f"{cluster}.{proc}"] = { + "ClusterId": cluster, + "ProcId": proc, + "JobStatus": 1, + "SUBMIT_Iwd": iwd, + "Out": _expand(out, cluster, proc), + "Err": _expand(err, cluster, proc), + "UserLog": f"{iwd}/{cluster}.{proc}.log", + } + return ok(f"{cluster}.0 - {cluster}.{count - 1}\n") + + @staticmethod + def _ads( + store: Mapping[str, dict[str, Any]], ids: list[str], args: list[str], empty: str + ) -> CommandResult: + wanted = args[args.index("-attributes") + 1].split(",") + found = [ + {k: v for k, v in store[i].items() if k in wanted} + for i in ids + if i in store + ] + return ok(json.dumps(found) if found else empty) + + def _remove(self, ids: list[str]) -> CommandResult: + lines, missing = [], False + for i in ids: + if i in self.queue: + ad = self.queue.pop(i) + self.history[i] = { + **ad, + "LastJobStatus": ad["JobStatus"], + "JobStatus": 3, + } + lines.append(f"Job {i} marked for removal") + else: + missing = True + if missing: + return CommandResult( + returncode=1, + stdout="\n".join(lines), + stderr="Couldn't find/remove all jobs\n", + ) + return ok("\n".join(lines)) + + def _transfer(self, ids: list[str]) -> CommandResult: + for i in ids: + ad = self.queue[i] + iwd = Path(ad["SUBMIT_Iwd"]) + iwd.mkdir(parents=True, exist_ok=True) + files = { + name: content + for name, content in [ + (ad.get("Out"), b"stdout of " + i.encode()), + (ad.get("Err"), b""), + (Path(ad["UserLog"]).name if "UserLog" in ad else None, b"log"), + ] + if name + } + for name, content in {**files, **self.outputs.get(i, {})}.items(): + (iwd / name).parent.mkdir(parents=True, exist_ok=True) + (iwd / name).write_bytes(content) + return ok("Fetching data files...\n") + + # -- helpers for the tests -------------------------------------------------- + + def _snapshot(self, call: Call) -> None: + """Record staged and credential files as they exist during the call.""" + for key in ("_CONDOR_SCITOKENS_FILE",): + if key in call.env: + self._record(call, Path(call.env[key])) + if call.argv[0] == "condor_submit": + submission = Path(call.argv[-1]).parent + for path in submission.rglob("*"): + if path.is_file(): + self._record(call, path) + + @staticmethod + def _record(call: Call, path: Path) -> None: + if path.is_file(): + call.files[str(path)] = ( + stat.S_IMODE(path.stat().st_mode), + path.read_bytes(), + ) + + def tools(self) -> list[str]: + return [call.argv[0] for call in self.calls] + + def finish(self, job: JobID | str, code: int = 4, **fields: Any) -> None: + self.queue[job] = {**self.queue[job], "JobStatus": code, **fields} + + +def _expand(name: str, cluster: int, proc: int) -> str: + return name.replace("$(ClusterId)", str(cluster)).replace("$(ProcId)", str(proc)) + + +def ok(stdout: str) -> CommandResult: + return CommandResult(returncode=0, stdout=stdout, stderr="") + + +def settings(**fields: Any) -> HTCondorCEConfig: + return HTCondorCEConfig(**{"kind": "htcondor-ce", "endpoint": ENDPOINT, **fields}) + + +def make( + condor: FakeCondor, + workdir: Path, + credentials: list[Any] | None = None, + **fields: Any, +) -> HTCondorCEBackend: + provider = StaticCredentials( + credentials if credentials is not None else [BearerToken(value=TOKEN)] + ) + return HTCondorCEBackend( + settings(**fields), provider, runner=condor, workdir=workdir + ) + + +def handle(job: str, **changes: str) -> JobHandle: + return JobHandle( + backend="htcondor-ce", resource=ENDPOINT, id=JobID(job) + ).model_copy(update=changes) diff --git a/tests/unit/backends/htcondorce/conftest.py b/tests/unit/backends/htcondorce/conftest.py new file mode 100644 index 0000000..688a77e --- /dev/null +++ b/tests/unit/backends/htcondorce/conftest.py @@ -0,0 +1,25 @@ +"""Fixtures for the HTCondor-CE backend tests.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from ._fake_condor import FakeCondor, make + + +@pytest.fixture +def condor() -> FakeCondor: + return FakeCondor() + + +@pytest.fixture +def workdir(tmp_path: Path) -> Path: + return tmp_path / "work" + + +@pytest.fixture +async def ce(condor: FakeCondor, workdir: Path): + async with make(condor, workdir) as backend: + yield backend diff --git a/tests/unit/backends/htcondorce/test_cleanup.py b/tests/unit/backends/htcondorce/test_cleanup.py new file mode 100644 index 0000000..2fe6547 --- /dev/null +++ b/tests/unit/backends/htcondorce/test_cleanup.py @@ -0,0 +1,90 @@ +"""The HTCondor-CE backend: kill and purge.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from intercede._backends.batch.protocols import CommandResult +from intercede._backends.monolith.htcondorce import HTCondorCEBackend + +from ._fake_condor import SPEC, FakeCondor, handle + + +class TestKill: + async def test_a_job_the_ce_no_longer_knows(self, ce: HTCondorCEBackend): + gone = handle("9.0") + assert (await ce.kill([gone]))[gone].reason == "unknown job" + assert (await ce.purge([gone]))[gone].reason == "unknown job" + + @pytest.mark.parametrize( + "fields", + [{"code": 4, "ExitCode": 0}, {"code": 4, "ExitCode": 3}], + ids=["done", "failed"], + ) + async def test_leaves_finished_jobs_and_their_output( + self, ce: HTCondorCEBackend, condor: FakeCondor, fields: dict[str, Any] + ): + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id, **fields) + condor.calls.clear() + outcome = (await ce.kill([h]))[h] + assert outcome.ok and outcome.reason == "already finished" + assert "condor_rm" not in condor.tools() + + async def test_condor_rm_refusing_one_job( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + sub = await ce.submit(SPEC, 2) + real = condor._remove + + def refuse_second(ids: list[str]) -> CommandResult: + result = real(ids[:1]) + return CommandResult( + returncode=1, + stdout=result.stdout, + stderr="Couldn't find/remove all jobs\n", + ) + + condor._remove = refuse_second # type: ignore[method-assign] + outcomes = await ce.kill(list(sub.handles.values())) + assert outcomes[sub.handles[0]].ok + assert not outcomes[sub.handles[1]].ok + assert ( + outcomes[sub.handles[1]].reason + == "condor_rm failed: Couldn't find/remove all jobs" + ) + + +class TestPurge: + async def test_a_job_removed_but_still_queued_is_already_released( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + # condor_rm returns before the schedd drops the job from the queue. + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id, code=3, LastJobStatus=4) + condor.calls.clear() + output = (await ce.get_output([h], tmp_path))[h] + assert ( + not output.ok and output.reason == "the job was purged; its output is gone" + ) + again = (await ce.purge([h]))[h] + assert again.ok and again.reason == "already released" + assert "condor_transfer_data" not in condor.tools() + assert "condor_rm" not in condor.tools() + + async def test_twice_is_harmless(self, ce: HTCondorCEBackend): + h = (await ce.submit(SPEC)).handles[0] + await ce.purge([h]) + again = (await ce.purge([h]))[h] + assert again.ok and again.reason == "already released" + + async def test_purges_active_jobs_too( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id, code=2) + assert (await ce.purge([h]))[h].ok + assert h.id not in condor.queue diff --git a/tests/unit/backends/htcondorce/test_construction.py b/tests/unit/backends/htcondorce/test_construction.py new file mode 100644 index 0000000..b9dce9f --- /dev/null +++ b/tests/unit/backends/htcondorce/test_construction.py @@ -0,0 +1,75 @@ +"""The HTCondor-CE backend: shape and construction.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from intercede import InterCEdeError, JobHandle, JobID, registry +from intercede._backends.monolith.htcondorce import HTCondorCEBackend +from intercede.auth import BearerToken, StaticCredentials +from intercede.protocols import Diagnosable + +from ._fake_condor import ENDPOINT, SPEC, TOKEN, FakeCondor, make, settings + + +class TestShape: + def test_no_live_diagnostics(self, condor: FakeCondor, workdir: Path): + assert not isinstance(make(condor, workdir), Diagnosable) + + def test_construction_does_no_io(self, condor: FakeCondor, workdir: Path): + make(condor, workdir) + assert not workdir.exists() + assert condor.calls == [] + + def test_built_by_the_registry(self): + built = registry.get_backend( + {"kind": "htcondor-ce", "endpoint": ENDPOINT}, StaticCredentials() + ) + assert isinstance(built, HTCondorCEBackend) + assert built.resource == ENDPOINT + + @pytest.mark.parametrize( + ("endpoint", "pool"), + [ + ("ce.example.org:9619", "ce.example.org:9619"), + ("ce.example.org", "ce.example.org:9619"), + ("ce:1234", "ce:1234"), + ], + ) + async def test_endpoint_becomes_pool_and_schedd_name( + self, condor: FakeCondor, workdir: Path, endpoint: str, pool: str + ): + backend = HTCondorCEBackend( + settings(endpoint=endpoint), + StaticCredentials([BearerToken(value=TOKEN)]), + runner=condor, + workdir=workdir, + ) + async with backend: + await backend.get_status( + [JobHandle(backend="htcondor-ce", resource=endpoint, id=JobID("1.0"))] + ) + argv = condor.calls[0].argv + assert argv[argv.index("-pool") + 1] == pool + assert argv[argv.index("-name") + 1] == pool.split(":")[0] + + async def test_schedd_overrides_the_name_not_the_pool( + self, condor: FakeCondor, workdir: Path + ): + async with make(condor, workdir, schedd="ce01@ce.example.org") as backend: + await backend.get_status( + [JobHandle(backend="htcondor-ce", resource=ENDPOINT, id=JobID("1.0"))] + ) + argv = condor.calls[0].argv + assert argv[argv.index("-pool") + 1] == ENDPOINT + assert argv[argv.index("-name") + 1] == "ce01@ce.example.org" + + async def test_closed_backend_refuses(self, condor: FakeCondor, workdir: Path): + backend = make(condor, workdir) + async with backend: + pass + await backend.__aexit__(None, None, None) # closing twice is harmless + with pytest.raises(InterCEdeError, match="closed"): + await backend.submit(SPEC) diff --git a/tests/unit/backends/htcondorce/test_output.py b/tests/unit/backends/htcondorce/test_output.py new file mode 100644 index 0000000..cf9ae63 --- /dev/null +++ b/tests/unit/backends/htcondorce/test_output.py @@ -0,0 +1,215 @@ +"""The HTCondor-CE backend: output.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from pathlib import Path + +import pytest + +from intercede._backends.batch.protocols import CommandResult +from intercede._backends.monolith.htcondorce import HTCondorCEBackend +from intercede.auth import BearerToken, StaticCredentials + +from ._fake_condor import SPEC, TOKEN, FakeCondor, make, ok, settings + + +class TestOutput: + async def test_collects_into_dest_with_a_manifest( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id) + condor.outputs[h.id] = {"result.txt": b"42", "sub/dir/data.bin": b"\x00"} + outcome = (await ce.get_output([h], tmp_path / "dest"))[h] + assert outcome.ok and outcome.value is not None + target = tmp_path / "dest" / "ce.example.org" / h.id + assert outcome.value.stdout == target / "1.0.out" + assert outcome.value.stdout.read_bytes() == b"stdout of 1.0" + assert outcome.value.stderr == target / "1.0.err" + assert outcome.value.log == target / "1.0.log" + assert outcome.value.files == { + "result.txt": target / "result.txt", + "sub/dir/data.bin": target / "sub/dir/data.bin", + } + + async def test_custom_stdout_name_is_recognised( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + h = ( + await ce.submit(SPEC.model_copy(update={"stdout": "logs/pilot.out"})) + ).handles[0] + condor.finish(h.id) + outcome = (await ce.get_output([h], tmp_path))[h] + assert outcome.value is not None and outcome.value.stdout is not None + assert outcome.value.stdout.name == "pilot.out" + assert "logs/pilot.out" not in outcome.value.files + + async def test_repeatable_and_replacing( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id) + condor.outputs[h.id] = {"old.txt": b"1"} + await ce.get_output([h], tmp_path) + condor.outputs[h.id] = {"new.txt": b"2"} + outcome = (await ce.get_output([h], tmp_path))[h] + assert outcome.value is not None and set(outcome.value.files) == {"new.txt"} + assert not (tmp_path / "ce.example.org" / h.id / "old.txt").exists() + + async def test_collection_area_is_cleaned( + self, ce: HTCondorCEBackend, condor: FakeCondor, workdir: Path, tmp_path: Path + ): + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id) + await ce.get_output([h], tmp_path / "dest") + assert not [p for p in workdir.rglob("*") if p.is_file()] + + @pytest.mark.parametrize("code", [1, 2], ids=["waiting", "running"]) + async def test_active_jobs( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path, code: int + ): + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id, code=code) + outcome = (await ce.get_output([h], tmp_path))[h] + assert not outcome.ok and outcome.reason == "job has not finished" + assert "condor_transfer_data" not in condor.tools() + + async def test_failed_jobs_can_be_collected( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id, ExitCode=3) # completed, payload exited non-zero + assert (await ce.get_output([h], tmp_path))[h].ok + + async def test_job_that_left_the_queue( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + h = (await ce.submit(SPEC)).handles[0] + await ce.purge([h]) + outcome = (await ce.get_output([h], tmp_path))[h] + assert ( + not outcome.ok + and outcome.reason == "the job has left the queue; its output is gone" + ) + + @pytest.mark.parametrize( + "iwd", ["/etc", "relative/dir", "{workdir}", "{workdir}/../escape"] + ) + async def test_the_ce_cannot_choose_where_we_write( + self, + ce: HTCondorCEBackend, + condor: FakeCondor, + workdir: Path, + tmp_path: Path, + iwd: str, + ): + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id, SUBMIT_Iwd=iwd.format(workdir=workdir)) + outcome = (await ce.get_output([h], tmp_path))[h] + assert ( + not outcome.ok + and outcome.reason + == "the job's output directory is outside our working area" + ) + assert "condor_transfer_data" not in condor.tools() + + async def test_transfer_failure_is_per_job( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id) + condor.fail["condor_transfer_data"] = CommandResult( + returncode=1, stdout="", stderr="ERROR: spool gone" + ) + outcome = (await ce.get_output([h], tmp_path))[h] + assert ( + not outcome.ok + and outcome.reason == "condor_transfer_data failed: ERROR: spool gone" + ) + + async def test_too_many_files( + self, condor: FakeCondor, workdir: Path, tmp_path: Path + ): + async with make(condor, workdir, max_output_files=3) as backend: + h = (await backend.submit(SPEC)).handles[0] + condor.finish(h.id) + condor.outputs[h.id] = {"a": b"", "b": b""} # plus stdout, stderr and log + outcome = (await backend.get_output([h], tmp_path))[h] + assert not outcome.ok and outcome.reason == "more than 3 output files" + + async def test_too_many_bytes( + self, condor: FakeCondor, workdir: Path, tmp_path: Path + ): + async with make(condor, workdir, max_output_bytes=10) as backend: + h = (await backend.submit(SPEC)).handles[0] + condor.finish(h.id) + condor.outputs[h.id] = {"big": b"x" * 100} + outcome = (await backend.get_output([h], tmp_path))[h] + assert not outcome.ok and outcome.reason == "output larger than 10 bytes" + + async def test_links_are_refused( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id) + real = condor._transfer + + def with_link(ids: list[str]) -> CommandResult: + result = real(ids) + (Path(condor.queue[h.id]["SUBMIT_Iwd"]) / "link").symlink_to("/etc/passwd") + return result + + condor._transfer = with_link # type: ignore[method-assign] + outcome = (await ce.get_output([h], tmp_path))[h] + assert not outcome.ok and "link" in (outcome.reason or "") + + async def test_transfer_uses_the_configured_timeout( + self, condor: FakeCondor, workdir: Path, tmp_path: Path + ): + timeouts: dict[str, float] = {} + + async def recording( + argv: Sequence[str], env: Mapping[str, str], timeout: float + ) -> CommandResult: + timeouts[argv[0]] = timeout + return await condor(argv, env, timeout) + + backend = HTCondorCEBackend( + settings(transfer_timeout_s=42), + StaticCredentials([BearerToken(value=TOKEN)]), + runner=recording, + workdir=workdir, + ) + async with backend: + h = (await backend.submit(SPEC)).handles[0] + condor.finish(h.id) + await backend.get_output([h], tmp_path) + assert timeouts["condor_transfer_data"] == 42 + assert timeouts["condor_submit"] == 120 + + +class TestOutputEdgeCases: + async def test_transfer_that_writes_nothing( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id) + condor._transfer = lambda ids: ok("Fetching data files...\n") # type: ignore[method-assign] + outcome = (await ce.get_output([h], tmp_path))[h] + assert ( + not outcome.ok + and outcome.reason == "condor_transfer_data returned no files" + ) + + async def test_log_outside_the_job_directory_is_not_the_log( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + h = (await ce.submit(SPEC)).handles[0] + condor.finish(h.id, UserLog="/var/log/elsewhere.log") + outcome = (await ce.get_output([h], tmp_path))[h] + assert outcome.ok and outcome.value is not None + assert outcome.value.log is None + assert ( + "elsewhere.log" in outcome.value.files + ) # written into the job directory by name diff --git a/tests/unit/backends/htcondorce/test_process.py b/tests/unit/backends/htcondorce/test_process.py new file mode 100644 index 0000000..127bf73 --- /dev/null +++ b/tests/unit/backends/htcondorce/test_process.py @@ -0,0 +1,207 @@ +"""The HTCondor-CE backend: how every tool runs, how failures map to errors, and the process runner.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +import pytest + +from intercede import ( + AuthenticationError, + ConfigurationError, + JobHandle, + JobID, + ResourceUnavailableError, +) +from intercede._backends.batch.protocols import CommandResult +from intercede._backends.monolith.htcondorce import HTCondorCEBackend +from intercede._backends.monolith.htcondorce.process import run_process +from intercede.auth import BearerToken, StaticCredentials + +from ._fake_condor import ENDPOINT, SPEC, TOKEN, FakeCondor, make, ok, settings + + +class TestEnvironment: + async def test_every_tool_runs_with_debug_for_error_classification( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + await ce.submit(SPEC) + assert all(call.argv[1] == "-debug" for call in condor.calls) + + async def test_authentication_is_required_and_token_based( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + await ce.submit(SPEC) + env = condor.calls[0].env + assert env["_CONDOR_SEC_CLIENT_AUTHENTICATION"] == "REQUIRED" + assert env["_CONDOR_SEC_CLIENT_AUTHENTICATION_METHODS"] == "SCITOKENS,IDTOKENS" + assert env["_CONDOR_TOOL_DEBUG"] == "D_SECURITY" + + async def test_token_is_a_private_file_only_during_the_call( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + await ce.submit(SPEC) + call = condor.calls[0] + token = call.env["_CONDOR_SCITOKENS_FILE"] + assert call.files[token] == (0o600, TOKEN.encode()) + assert not Path(token).exists() + assert not Path(call.env["_CONDOR_SEC_TOKEN_DIRECTORY"]).exists() + assert not Path(call.env["_CONDOR_LOG"]).exists() + + async def test_no_config_file_unless_the_operator_set_one( + self, ce: HTCondorCEBackend, condor: FakeCondor, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.delenv("CONDOR_CONFIG", raising=False) + await ce.get_status( + [JobHandle(backend="htcondor-ce", resource=ENDPOINT, id=JobID("1.0"))] + ) + assert condor.calls[-1].env["CONDOR_CONFIG"] == "ONLY_ENV" + monkeypatch.setenv("CONDOR_CONFIG", "/etc/condor/site.conf") + await ce.get_status( + [JobHandle(backend="htcondor-ce", resource=ENDPOINT, id=JobID("1.0"))] + ) + assert condor.calls[-1].env["CONDOR_CONFIG"] == "/etc/condor/site.conf" + + async def test_ca_directory_from_x509_cert_dir( + self, ce: HTCondorCEBackend, condor: FakeCondor, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setenv("X509_CERT_DIR", "/etc/grid-security/certificates") + await ce.submit(SPEC) + assert ( + condor.calls[0].env["_CONDOR_AUTH_SSL_CLIENT_CADIR"] + == "/etc/grid-security/certificates" + ) + + async def test_scopes_follow_the_operation(self, condor: FakeCondor, workdir: Path): + asked: list[frozenset[str]] = [] + + class Recording(StaticCredentials): + async def get(self, requirements: Any) -> Any: + asked.append(requirements.scopes) + return await super().get(requirements) + + handle = JobHandle(backend="htcondor-ce", resource=ENDPOINT, id=JobID("1.0")) + backend = HTCondorCEBackend( + settings(), + Recording([BearerToken(value=TOKEN)]), + runner=condor, + workdir=workdir, + ) + async with backend: + await backend.submit(SPEC) + await backend.get_status([handle]) + await backend.kill([handle]) + assert asked[:2] == [frozenset({"compute.create"}), frozenset({"compute.read"})] + + +class TestErrors: + @pytest.mark.parametrize( + "stderr", + [ + "10/21/26 13:52:19 AUTHENTICATE:1003:Failed to authenticate with any method\n" + "Error: Couldn't contact the condor_collector", + "PERMISSION DENIED to submituser", + # condor_submit with a rejected token (HTCondor 25): no AUTHENTICATE + # summary, and an error that looks like an unreachable schedd. + "10/26/26 11:37:15 AUTHENTICATE: method 2048 (IDTOKENS) failed.\n" + "10/26/26 11:37:15 SECMAN: required authentication with collector " + "127.0.0.1:9619 failed, so aborting command QUERY_SCHEDD_ADS.\n" + "ERROR: Can't find address of schedd htcondor-ce@backend", + ], + ids=["token-rejected", "permission-denied", "submit-token-rejected"], + ) + async def test_authentication_failure( + self, ce: HTCondorCEBackend, condor: FakeCondor, stderr: str + ): + condor.fail["condor_submit"] = CommandResult( + returncode=1, stdout="", stderr=stderr + ) + with pytest.raises(AuthenticationError): + await ce.submit(SPEC) + + @pytest.mark.parametrize( + "stderr", + [ + "10/21/26 13:52:19 connect errno = 111\nError: Couldn't contact the condor_collector on ce:9619.", + "Can't find address for schedd ce.example.org", + ], + ids=["refused", "no-address"], + ) + async def test_unreachable( + self, ce: HTCondorCEBackend, condor: FakeCondor, stderr: str + ): + condor.fail["condor_q"] = CommandResult(returncode=1, stdout="", stderr=stderr) + with pytest.raises(ResourceUnavailableError, match="could not be reached"): + await ce.get_status( + [JobHandle(backend="htcondor-ce", resource=ENDPOINT, id=JobID("1.0"))] + ) + + async def test_other_failures_name_the_tool_without_debug_lines( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + condor.fail["condor_submit"] = CommandResult( + returncode=1, + stdout="", + stderr="10/21/26 13:52:19 SECMAN: noise\nERROR: on Line 3 of submit file\n", + ) + with pytest.raises( + ResourceUnavailableError, + match="condor_submit failed: ERROR: on Line 3 of submit file$", + ): + await ce.submit(SPEC) + + @pytest.mark.parametrize("raw", ["Submitting job(s).", "1.0 - 1.0\n2.0 - 2.0"]) + async def test_unreadable_submit_output( + self, ce: HTCondorCEBackend, condor: FakeCondor, raw: str + ): + condor.fail["condor_submit"] = ok(raw) + with pytest.raises( + ResourceUnavailableError, match="unexpected condor_submit output" + ): + await ce.submit(SPEC) + + async def test_job_count_mismatch(self, ce: HTCondorCEBackend, condor: FakeCondor): + condor.fail["condor_submit"] = ok("5.0 - 5.0") + with pytest.raises( + ResourceUnavailableError, match="created 1 jobs, expected 2" + ): + await ce.submit(SPEC, 2) + + async def test_unreadable_query_output( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + condor.fail["condor_q"] = ok("not json") + with pytest.raises(ResourceUnavailableError, match="unreadable output"): + await ce.get_status( + [JobHandle(backend="htcondor-ce", resource=ENDPOINT, id=JobID("1.0"))] + ) + + async def test_no_token_from_the_provider(self, condor: FakeCondor, workdir: Path): + async with make(condor, workdir, credentials=[]) as backend: + with pytest.raises(AuthenticationError, match="BearerToken"): + await backend.submit(SPEC) + assert condor.calls == [] + + +class TestRunProcess: + async def test_runs_a_command_with_only_the_given_environment(self): + result = await run_process( + ["/bin/sh", "-c", "echo $ONLY; echo err >&2; exit 3"], + {"ONLY": "this", "PATH": os.environ["PATH"]}, + 10, + ) + assert (result.returncode, result.stdout, result.stderr) == ( + 3, + "this\n", + "err\n", + ) + + async def test_missing_tool_is_a_configuration_error(self): + with pytest.raises(ConfigurationError, match="HTCondor client tools"): + await run_process(["condor_definitely_not_installed"], dict(os.environ), 10) + + async def test_timeout_kills_the_tool(self): + with pytest.raises(ResourceUnavailableError, match="did not finish"): + await run_process(["/bin/sleep", "10"], dict(os.environ), 0.2) diff --git a/tests/unit/backends/htcondorce/test_status.py b/tests/unit/backends/htcondorce/test_status.py new file mode 100644 index 0000000..2696229 --- /dev/null +++ b/tests/unit/backends/htcondorce/test_status.py @@ -0,0 +1,54 @@ +"""The HTCondor-CE backend: status.""" + +from __future__ import annotations + +import re + +import pytest + +from intercede import JobStatus +from intercede._backends.monolith.htcondorce import HTCondorCEBackend + +from ._fake_condor import SPEC, FakeCondor, handle + + +class TestStatus: + async def test_queue_then_history_for_the_rest( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + sub = await ce.submit(SPEC, 2) + condor.finish("1.0") + await ce.purge([sub.handles[1]]) + condor.calls.clear() + states = await ce.get_status(list(sub.handles.values())) + assert states == { + sub.handles[0]: JobStatus.DONE, + sub.handles[1]: JobStatus.ABORTED, + } + assert condor.tools() == ["condor_q", "condor_history"] + history_ids = [a for a in condor.calls[1].argv if re.fullmatch(r"\d+\.\d+", a)] + assert history_ids == ["1.1"] # only what the queue did not know + + @pytest.mark.parametrize( + "foreign", + [ + {"resource": "other.example.org:9619"}, + {"backend": "arc"}, + {"id": "-constraint true"}, + {"id": "1"}, + {"id": "1.0 1.1"}, + ], + ids=["other-ce", "other-backend", "option-injection", "no-proc", "two-ids"], + ) + async def test_foreign_and_malformed_handles_never_reach_a_command( + self, ce: HTCondorCEBackend, condor: FakeCondor, foreign: dict[str, str] + ): + bad = handle("1.0", **foreign) + assert await ce.get_status([bad]) == {bad: JobStatus.UNKNOWN} + assert condor.calls == [] + + async def test_large_polls_are_chunked( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + await ce.get_status([handle(f"1.{i}") for i in range(250)]) + assert condor.tools() == ["condor_q"] * 3 + ["condor_history"] * 3 diff --git a/tests/unit/backends/htcondorce/test_submit.py b/tests/unit/backends/htcondorce/test_submit.py new file mode 100644 index 0000000..4ce5c37 --- /dev/null +++ b/tests/unit/backends/htcondorce/test_submit.py @@ -0,0 +1,311 @@ +"""The HTCondor-CE backend: submit descriptions, outputs and containers.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from pathlib import Path + +import pytest + +from intercede import ( + ContainerSpec, + CopyVars, + FileRef, + OutputMember, + OutputSpec, + ResourceUnavailableError, + SpecificationRejectedError, + SubmissionSpec, +) +from intercede._backends.batch.protocols import CommandResult +from intercede._backends.monolith.htcondorce import HTCondorCEBackend +from intercede.auth import BearerToken, StaticCredentials + +from ._fake_condor import SPEC, TOKEN, FakeCondor, make, settings + + +class TestSubmit: + async def test_a_secret_named_like_an_input_is_refused_before_any_call( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + source = tmp_path / "pilot.cfg" + source.write_text("data") + spec = SPEC.model_copy( + update={"inputs": [FileRef(name="pilot.cfg", source=str(source))]} + ) + with pytest.raises(SpecificationRejectedError, match="pilot.cfg"): + await ce.submit(spec, [CopyVars(secrets={"pilot.cfg": b"s"})]) + assert condor.calls == [] + + async def test_zero_copies_submit_nothing( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + assert (await ce.submit(SPEC, 0)).handles == {} + assert condor.calls == [] + + async def test_local_inputs_keep_their_directories( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + source = tmp_path / "anything.dat" + source.write_text("data") + spec = SPEC.model_copy( + update={"inputs": [FileRef(name="sub/dir/in.txt", source=str(source))]} + ) + await ce.submit(spec) + staged = [ + p for p in condor.calls[0].files if p.endswith("inputs/sub/dir/in.txt") + ] + assert len(staged) == 1 and condor.calls[0].files[staged[0]][1] == b"data" + + async def test_secrets_are_private_and_gone_afterwards( + self, ce: HTCondorCEBackend, condor: FakeCondor, workdir: Path + ): + copies = [ + CopyVars(secrets={"pilot.secret": b"s0"}), + CopyVars(secrets={"pilot.secret": b"s1"}), + ] + await ce.submit(SPEC, copies) + secrets = { + path: entry + for path, entry in condor.calls[0].files.items() + if path.endswith("pilot.secret") + } + assert sorted(entry for entry in secrets.values()) == [ + (0o600, b"s0"), + (0o600, b"s1"), + ] + assert not [p for p in workdir.rglob("*") if p.is_file()] + + async def test_secret_never_in_the_description( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + await ce.submit(SPEC, [CopyVars(secrets={"pilot.secret": b"TOPSECRET"})]) + assert "TOPSECRET" not in condor.descriptions[0] + + async def test_different_variable_names_get_separate_clusters( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + # HTCondor merges each process's environment onto the previous one, so + # a variable of one copy would leak into the next in the same cluster. + copies = [ + CopyVars(environment={"STAMP": "a"}), + CopyVars(environment={"STAMP": "b", "EXTRA": "x"}), + CopyVars(environment={"STAMP": "c"}), + ] + sub = await ce.submit(SPEC, copies) + assert condor.tools().count("condor_submit") == 2 + clusters = {i: h.id.split(".")[0] for i, h in sub.handles.items()} + assert clusters[0] == clusters[2] != clusters[1] + assert "EXTRA" not in condor.descriptions[0] + + async def test_a_later_cluster_failing_keeps_the_submitted_copies( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + copies = [CopyVars(environment={"A": "1"}), CopyVars(environment={"B": "2"})] + submitted = 0 + real = condor.__call__ + + async def second_submit_fails( + argv: Sequence[str], env: Mapping[str, str], timeout: float + ) -> CommandResult: + nonlocal submitted + if argv[0] == "condor_submit": + submitted += 1 + if submitted == 2: + return CommandResult( + returncode=1, stdout="", stderr="ERROR: schedd full" + ) + return await real(argv, env, timeout) + + backend = HTCondorCEBackend( + settings(), + StaticCredentials([BearerToken(value=TOKEN)]), + runner=second_submit_fails, + workdir=ce._workdir, + ) + async with backend: + sub = await backend.submit(SPEC, copies) + assert sorted(sub.handles) == [0] + assert sub.failures == {1: "condor_submit failed: ERROR: schedd full"} + + async def test_nothing_submitted_raises( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + condor.fail["condor_submit"] = CommandResult( + returncode=1, stdout="", stderr="ERROR: schedd full" + ) + with pytest.raises(ResourceUnavailableError, match="schedd full"): + await ce.submit(SPEC, 2) + + async def test_url_inputs_are_left_to_the_resource( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + spec = SPEC.model_copy( + update={ + "inputs": [ + FileRef(name="in.dat", source="https://example.org/data/in.dat?x=1") + ] + } + ) + await ce.submit(spec) + assert "https://example.org/data/in.dat?x=1" in condor.descriptions[0] + + async def test_url_input_under_another_name_is_refused( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + spec = SPEC.model_copy( + update={ + "inputs": [ + FileRef(name="renamed.dat", source="https://example.org/in.dat") + ] + } + ) + with pytest.raises(SpecificationRejectedError, match="another name"): + await ce.submit(spec) + assert "condor_submit" not in condor.tools() + + async def test_missing_local_input_is_refused( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + spec = SPEC.model_copy( + update={"inputs": [FileRef(name="in", source=str(tmp_path / "missing"))]} + ) + with pytest.raises(SpecificationRejectedError, match="cannot read"): + await ce.submit(spec) + assert "condor_submit" not in condor.tools() + + async def test_command_on_the_worker_is_not_shipped( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + # Not a file on this host, so it is a command on the worker node. + await ce.submit( + SPEC.model_copy(update={"executable": "/opt/site/bin/run-payload"}) + ) + assert "transfer_executable = false" in condor.descriptions[0] + + async def test_resources_and_queue(self, condor: FakeCondor, workdir: Path): + spec = SPEC.model_copy( + update={ + "resources": SPEC.resources.model_copy( + update={"cpus": 4, "memory_mb": 8000, "wall_time_s": 3600} + ) + } + ) + async with make(condor, workdir, queue="default") as backend: + await backend.submit(spec) + text = condor.descriptions[0] + assert all( + line in text + for line in [ + "+xcount = 4", + "+maxMemory = 8000", + "+BatchRuntime = 3600", + '+queue = "default"', + ] + ) + + async def test_spec_queue_wins_over_configured_queue( + self, condor: FakeCondor, workdir: Path + ): + spec = SPEC.model_copy( + update={"resources": SPEC.resources.model_copy(update={"queue": "short"})} + ) + async with make(condor, workdir, queue="default") as backend: + await backend.submit(spec) + assert '+queue = "short"' in condor.descriptions[0] + + +class TestSubmitOutputs: + async def test_kept_and_uploaded_members( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + outputs = OutputSpec( + members=[ + OutputMember(pattern="kept.txt"), + OutputMember(pattern="sent.root", destination="https://se/sent.root"), + ] + ) + await ce.submit(SPEC.model_copy(update={"outputs": outputs})) + text = condor.descriptions[0] + assert "transfer_output_files = kept.txt, sent.root" in text + assert 'transfer_output_remaps = "sent.root = https://se/sent.root"' in text + + async def test_include_unlisted_brings_back_everything( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + outputs = OutputSpec( + members=[OutputMember(pattern="*.root")], include_unlisted=True + ) + await ce.submit(SPEC.model_copy(update={"outputs": outputs})) + assert "transfer_output_files" not in condor.descriptions[0] + + @pytest.mark.parametrize( + "member", + [ + OutputMember(pattern="*.root"), + OutputMember(pattern="out?.txt"), + OutputMember(pattern="[ab].txt"), + ], + ) + async def test_patterns_are_refused_without_include_unlisted( + self, ce: HTCondorCEBackend, member: OutputMember + ): + with pytest.raises(SpecificationRejectedError, match="by name, not by pattern"): + await ce.submit( + SPEC.model_copy(update={"outputs": OutputSpec(members=[member])}) + ) + + async def test_a_pattern_cannot_be_uploaded(self, ce: HTCondorCEBackend): + member = OutputMember(pattern="*.root", destination="https://se/") + with pytest.raises(SpecificationRejectedError, match="by name"): + await ce.submit( + SPEC.model_copy( + update={ + "outputs": OutputSpec(members=[member], include_unlisted=True) + } + ) + ) + + +class TestContainers: + async def test_refused_without_a_launcher( + self, ce: HTCondorCEBackend, condor: FakeCondor + ): + with pytest.raises(SpecificationRejectedError, match="no container_launcher"): + await ce.submit( + SPEC.model_copy( + update={"container": ContainerSpec(image="docker://alpine")} + ) + ) + assert condor.calls == [] + + async def test_wrapped_by_the_configured_launcher( + self, condor: FakeCondor, workdir: Path + ): + spec = SubmissionSpec( + executable="/usr/bin/run", + arguments=["x"], + container=ContainerSpec(image="/cvmfs/img.sif"), + ) + async with make( + condor, workdir, container_launcher=["apptainer", "exec", "{image}"] + ) as backend: + await backend.submit(spec) + text = condor.descriptions[0] + assert "executable = apptainer" in text + assert "transfer_executable = false" in text + assert "arguments = \"'exec' '/cvmfs/img.sif' '/usr/bin/run' 'x'\"" in text + + async def test_a_local_file_cannot_run_inside_an_image( + self, condor: FakeCondor, workdir: Path, tmp_path: Path + ): + script = tmp_path / "s.sh" + script.write_text("") + spec = SubmissionSpec( + executable=str(script), container=ContainerSpec(image="docker://a") + ) + async with make( + condor, workdir, container_launcher=["apptainer", "exec", "{image}"] + ) as backend: + with pytest.raises(SpecificationRejectedError, match="inside the image"): + await backend.submit(spec) diff --git a/tests/unit/backends/htcondorce/test_working_area.py b/tests/unit/backends/htcondorce/test_working_area.py new file mode 100644 index 0000000..5e59070 --- /dev/null +++ b/tests/unit/backends/htcondorce/test_working_area.py @@ -0,0 +1,99 @@ +"""The HTCondor-CE backend: the working area, and keeping nothing between calls.""" + +from __future__ import annotations + +import stat +from collections.abc import Mapping, Sequence +from pathlib import Path + +import pytest + +from intercede import ConfigurationError, JobStatus +from intercede._backends.batch.protocols import CommandResult +from intercede._backends.monolith.htcondorce import HTCondorCEBackend +from intercede.auth import BearerToken, StaticCredentials + +from ._fake_condor import SPEC, TOKEN, FakeCondor, make, settings + + +class TestWorkingArea: + async def test_is_private(self, ce: HTCondorCEBackend, workdir: Path): + await ce.submit(SPEC) + assert stat.S_IMODE(workdir.stat().st_mode) == 0o700 + + async def test_a_link_in_its_place_is_refused( + self, condor: FakeCondor, tmp_path: Path + ): + target = tmp_path / "elsewhere" + target.mkdir() + link = tmp_path / "work" + link.symlink_to(target) + async with make(condor, link) as backend: + with pytest.raises( + ConfigurationError, match="not a directory owned by this user" + ): + await backend.submit(SPEC) + + +class TestStateless: + """The backend keeps nothing about jobs between calls.""" + + async def test_no_job_is_remembered_after_collection( + self, ce: HTCondorCEBackend, condor: FakeCondor, tmp_path: Path + ): + import gc + + sub = await ce.submit(SPEC, 3) + for h in sub.handles.values(): + condor.finish(h.id) + await ce.get_output(list(sub.handles.values()), tmp_path) + gc.collect() + assert len(ce._collecting) == 0 + + async def test_one_job_collected_twice_at_once( + self, condor: FakeCondor, workdir: Path, tmp_path: Path + ): + import asyncio + + transfers: list[str] = [] # "start" / "end", to see whether they overlap + + async def slow_transfer( + argv: Sequence[str], env: Mapping[str, str], timeout: float + ) -> CommandResult: + if argv[0] != "condor_transfer_data": + return await condor(argv, env, timeout) + transfers.append("start") + await asyncio.sleep(0.05) # the second collection arrives meanwhile + result = await condor(argv, env, timeout) + transfers.append("end") + return result + + provider = StaticCredentials([BearerToken(value=TOKEN)]) + backend = HTCondorCEBackend( + settings(), provider, runner=slow_transfer, workdir=workdir + ) + async with backend: + h = (await backend.submit(SPEC)).handles[0] + condor.finish(h.id) + condor.outputs[h.id] = {"result.txt": b"42"} + first, second = await asyncio.gather( + backend.get_output([h], tmp_path / "a"), + backend.get_output([h], tmp_path / "b"), + ) + # Both succeed, one after the other: never two transfers into one directory. + assert transfers == ["start", "end", "start", "end"] + for outcome in (first[h], second[h]): + assert outcome.ok and outcome.value is not None + assert outcome.value.files["result.txt"].read_bytes() == b"42" + + async def test_a_new_backend_knows_the_jobs_of_another( + self, condor: FakeCondor, workdir: Path, tmp_path: Path + ): + # Everything needed lives in the handle and at the CE, not in the backend. + async with make(condor, workdir) as first: + h = (await first.submit(SPEC)).handles[0] + condor.finish(h.id, ExitCode=0) + async with make(condor, workdir) as second: + assert (await second.get_status([h]))[h] == JobStatus.DONE + assert (await second.get_output([h], tmp_path))[h].ok + assert (await second.purge([h]))[h].ok diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index b243067..e3b6575 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -13,11 +13,12 @@ from intercede import JobBackend from intercede._backends.config import FakeConfig +from intercede.auth import StaticCredentials from intercede.registry import _build # Built through the registry, so the suite also covers the registry path. BACKENDS: dict[str, Callable[[], JobBackend]] = { - "fake": lambda: _build(FakeConfig()), + "fake": lambda: _build(FakeConfig(), StaticCredentials()), } diff --git a/tests/unit/test_auth.py b/tests/unit/test_auth.py new file mode 100644 index 0000000..3977caa --- /dev/null +++ b/tests/unit/test_auth.py @@ -0,0 +1,112 @@ +"""Credential values, requirements and the static provider.""" + +from __future__ import annotations + +from collections.abc import Callable +from datetime import datetime +from typing import Any + +import pytest +from pydantic import ValidationError + +from intercede.auth import ( + BearerToken, + Credential, + CredentialProvider, + CredentialRequirements, + SSHPassword, + SSHPrivateKey, + StaticCredentials, +) +from intercede.exceptions import AuthenticationError + +SECRET = "TOPSECRET" # noqa: S105 (a fake secret the tests look for) + +# One instance of every kind, each holding SECRET in all of its secret fields. +EVERY_KIND: dict[type[Credential], Credential] = { + BearerToken: BearerToken(value=SECRET), + SSHPrivateKey: SSHPrivateKey(pem=SECRET.encode(), passphrase=SECRET), + SSHPassword: SSHPassword(value=SECRET), +} + + +@pytest.mark.parametrize("kind", list(EVERY_KIND), ids=lambda kind: kind.__name__) +def test_a_secret_never_shows(kind: type[Credential]): + credential = EVERY_KIND[kind] + for shown in ( + repr(credential), + str(credential), + credential.model_dump_json(), + repr(StaticCredentials([credential])), + ): + assert SECRET not in shown + + +@pytest.mark.parametrize( + "build", + [ + lambda: BearerToken(value="t", token=SECRET), + lambda: SSHPrivateKey(pem=b"k", passphrase=[SECRET]), + lambda: StaticCredentials([{"value": SECRET}]), + ], + ids=["unknown-field", "wrong-type", "plain-data-in-provider"], +) +def test_a_mistake_never_repeats_the_secret(build: Callable[[], Any]): + with pytest.raises(ValueError) as exc_info: # also pydantic's ValidationError + build() + assert SECRET not in str(exc_info.value) + + +def test_expiry_must_have_a_timezone(): + # A naive time could not be compared with the current time in UTC. + with pytest.raises(ValidationError, match="timezone"): + BearerToken(value="t", expires_at=datetime(2030, 1, 1)) # noqa: DTZ001 + + +class TestCredentialRequirements: + def test_names_one_kind(self): + requirements = CredentialRequirements(resource="r", kind=BearerToken) + assert requirements.kind is BearerToken + + def test_needs_a_kind(self): + with pytest.raises(ValidationError, match="kind"): + CredentialRequirements(resource="r") # type: ignore[call-arg] + + @pytest.mark.parametrize("kind", [Credential, str, "BearerToken"]) + def test_the_kind_is_a_concrete_credential_class(self, kind: Any): + with pytest.raises(ValidationError): + CredentialRequirements(resource="r", kind=kind) + + +class TestStaticCredentials: + def test_is_a_provider(self): + assert isinstance(StaticCredentials(), CredentialProvider) + + async def test_hands_out_only_the_requested_kind(self): + token, password = BearerToken(value="t"), SSHPassword(value="pw") + provider = StaticCredentials([token, password]) + requirements = CredentialRequirements(resource="ce:9619", kind=BearerToken) + async with provider: + assert await provider.get(requirements) is token + + async def test_a_kind_it_does_not_hold_is_named(self): + # A token and a password both have a single "value" field: it goes by + # kind, not by shape. + provider = StaticCredentials([SSHPassword(value="pw")]) + requirements = CredentialRequirements(resource="ce:9619", kind=BearerToken) + with pytest.raises(AuthenticationError, match="no BearerToken for ce:9619"): + await provider.get(requirements) + + @pytest.mark.parametrize( + "credentials", + [ + [BearerToken(value="a"), BearerToken(value="b")], + [{"value": "x"}], + ["token"], + [Credential()], + ], + ids=["two-of-a-kind", "plain-data", "a-string", "the-base-class"], + ) + def test_refuses_bad_input_at_construction(self, credentials: Any): + with pytest.raises(ValueError): + StaticCredentials(credentials) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 2856b53..a89b203 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -64,7 +64,14 @@ def test_defaults_are_filled_in(): assert (ssh.port, ssh.auth, ssh.connect_timeout_s) == (22, "key", 30) assert ssh.transfer_timeout_s == 3600 ce = validate_config(CE) - assert isinstance(ce, HTCondorCEConfig) and ce.auth == {"token"} + assert isinstance(ce, HTCondorCEConfig) and ce.auth == "token" + arc = validate_config(ARC) + assert isinstance(arc, ARCConfig) and arc.auth == "token" + + +def test_an_htcondor_ce_can_name_its_schedd(): + ce = validate_config({**CE, "schedd": "ce01@ce.example.org"}) + assert isinstance(ce, HTCondorCEConfig) and ce.schedd == "ce01@ce.example.org" def test_a_model_passes_through_unchanged(): @@ -127,9 +134,10 @@ def test_every_missing_field_is_named_at_once(self): @pytest.mark.parametrize( ("data", "message"), [ - ({**ARC, "auth": []}, "auth must name at least one"), - ({**CE, "auth": ["proxy"]}, "must include 'token'"), - ({**ARC, "auth": ["key"]}, "auth"), + ({**ARC, "auth": "proxy"}, "auth: Input should be 'token'"), + ({**CE, "auth": "proxy"}, "auth: Input should be 'token'"), + ({**CE, "auth": ["token", "proxy"]}, "auth: Input should be 'token'"), + ({**ARC, "auth": "key"}, "auth: Input should be 'token'"), ({**SSH, "auth": "token"}, "auth"), ( {**SSH, "known_hosts": "\n\t "}, @@ -139,14 +147,19 @@ def test_every_missing_field_is_named_at_once(self): {**LOCAL, "container_launcher": ["apptainer", "exec"]}, "'{image}' placeholder", ), + ({**CE, "endpoint": "https://ce.example.org"}, "endpoint must be 'host'"), + ({**CE, "schedd": "-name evil"}, "schedd must be 'host' or 'name@host'"), ({**CLOUD, "driver_options": {"OS_PASSWORD": "x"}}, "credential provider"), ({**LOCAL, "max_output_files": 0}, "max_output_files"), ], ids=[ - "arc-no-auth", - "htcondor-ce-no-token", + "arc-no-proxy", + "htcondor-ce-no-proxy", + "htcondor-ce-one-credential", "ce-ssh-auth", "ssh-ce-auth", + "ce-endpoint-url", + "ce-schedd-option", "no-host-keys", "launcher-no-image", "secret-driver-option", @@ -168,6 +181,9 @@ def test_an_unknown_field(self): def test_a_wrong_value(self): assert "toto4" not in rejected(**SSH, port="toto4") + def test_a_value_that_breaks_a_rule(self): + assert "toto4" not in rejected(**{**CE, "endpoint": "https://toto4"}) + def test_building_a_model_directly(self): with pytest.raises(ValueError) as exc_info: BatchSSHConfig(**SSH, password="toto4") # type: ignore[call-arg] # noqa: S106 diff --git a/tests/unit/test_conformance.py b/tests/unit/test_conformance.py index 5d6f51e..ae309fa 100644 --- a/tests/unit/test_conformance.py +++ b/tests/unit/test_conformance.py @@ -100,13 +100,23 @@ async def test_kill_reports_per_job(backend: JobBackend): assert outcome[theirs].reason -async def test_purge_forgets_the_job(backend: JobBackend): +async def test_purge_releases_the_job(backend: JobBackend, tmp_path: Path): + """Purge releases a job's output and remote state. + + Afterwards the job is no longer active and its output cannot be collected. + Whether its status is then UNKNOWN or a final state depends on whether the + resource still remembers it: a schedd's history does, an in-memory fake + does not. Both are within the contract. + """ if not isinstance(backend, Purgeable): pytest.skip("backend cannot purge") sub = await backend.submit(SPEC) handle = sub.handles[0] assert (await backend.purge([handle]))[handle].ok - assert (await backend.get_status([handle]))[handle] == JobStatus.UNKNOWN + status = (await backend.get_status([handle]))[handle] + assert status not in (JobStatus.SUBMITTED, JobStatus.WAITING, JobStatus.RUNNING) + if isinstance(backend, OutputRetriever): + assert not (await backend.get_output([handle], tmp_path))[handle].ok async def test_output_of_an_unknown_job_fails_per_job( @@ -138,3 +148,12 @@ async def test_a_secret_named_like_an_input_is_refused( clashing = [CopyVars(), CopyVars(secrets={"pilot.cfg": b"s"})] with pytest.raises(SpecificationRejectedError, match="pilot.cfg"): await backend.submit(spec, clashing) + + +async def test_a_missing_local_input_is_refused(backend: JobBackend, tmp_path: Path): + spec = SubmissionSpec( + executable="/bin/true", + inputs=[FileRef(name="in", source=str(tmp_path / "missing"))], + ) + with pytest.raises(SpecificationRejectedError, match="cannot read"): + await backend.submit(spec) diff --git a/tests/unit/test_credentials.py b/tests/unit/test_credentials.py new file mode 100644 index 0000000..ab45d13 --- /dev/null +++ b/tests/unit/test_credentials.py @@ -0,0 +1,196 @@ +"""When a backend asks its provider again, and what it accepts back.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from types import TracebackType +from typing import Any, Self + +import pytest + +from intercede import AuthenticationError, InterCEdeError, validate_config +from intercede._backends.credentials import ( + DEFAULT_MARGIN, + credential_requirements, + is_stale, + refresh_if_stale, +) +from intercede.auth import ( + BearerToken, + Credential, + CredentialRequirements, + SSHPassword, + SSHPrivateKey, +) + +NOW = datetime(2030, 1, 1, 12, 0, tzinfo=UTC) +TOKEN_ONLY = CredentialRequirements(resource="ce:9619", kind=BearerToken) + + +def token(expires_in: timedelta | None = None) -> BearerToken: + return BearerToken( + value="t", expires_at=None if expires_in is None else NOW + expires_in + ) + + +class Recording: + """A provider that returns whatever the test gives it, and counts calls.""" + + def __init__(self, result: Any = None, error: BaseException | None = None) -> None: + self.result = result + self.error = error + self.asked: list[CredentialRequirements] = [] + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: ... + + async def get(self, requirements: CredentialRequirements) -> Any: + self.asked.append(requirements) + if self.error is not None: + raise self.error + return self.result + + +class TestIsStale: + @pytest.mark.parametrize( + ("credential", "stale"), + [ + (None, True), # nothing asked for yet + (token(), False), # never expires + (token(DEFAULT_MARGIN + timedelta(seconds=1)), False), + (token(DEFAULT_MARGIN), True), # inside the margin + (token(-timedelta(hours=1)), True), # expired + (SSHPassword(value="p"), True), # not what is required + ], + ids=[ + "none", + "never-expires", + "outside-margin", + "at-margin", + "expired", + "another-kind", + ], + ) + def test_one_credential(self, credential: Credential | None, stale: bool): + assert is_stale(credential, TOKEN_ONLY, now=NOW) is stale + + +class TestRefreshIfStale: + async def test_fresh_credential_is_kept_without_asking(self): + current = token(timedelta(hours=1)) + provider = Recording() + assert await refresh_if_stale(current, TOKEN_ONLY, provider, now=NOW) is current + assert provider.asked == [] + + async def test_stale_credential_is_replaced(self): + fresh = token(timedelta(hours=1)) + provider = Recording(fresh) + current = token(timedelta(minutes=1)) + assert await refresh_if_stale(current, TOKEN_ONLY, provider, now=NOW) is fresh + assert provider.asked == [TOKEN_ONLY] + + async def test_short_lived_credentials_are_accepted(self): + # Inside the margin but not expired: usable now, asked again next time. + fresh = token(timedelta(minutes=1)) + assert ( + await refresh_if_stale(None, TOKEN_ONLY, Recording(fresh), now=NOW) is fresh + ) + + +class TestProviderFailures: + """Every failure is a whole-operation AuthenticationError.""" + + async def test_provider_raising_is_an_authentication_error(self): + provider = Recording(error=RuntimeError("no token for s3cret-url")) + with pytest.raises(AuthenticationError) as exc_info: + await refresh_if_stale(None, TOKEN_ONLY, provider, now=NOW) + assert isinstance(exc_info.value.__cause__, RuntimeError) + message = str(exc_info.value) + assert "ce:9619" in message and "RuntimeError" in message + assert "s3cret" not in message # the provider's text may hold a secret + + async def test_our_own_errors_pass_through_unchanged(self): + error = InterCEdeError("x") + with pytest.raises(InterCEdeError) as exc_info: + await refresh_if_stale(None, TOKEN_ONLY, Recording(error=error), now=NOW) + assert exc_info.value is error + + @pytest.mark.parametrize( + ("result", "returned"), + [ + (None, "NoneType"), + ([token()], "list"), + (SSHPassword(value="p"), "SSHPassword"), + ], + ids=["nothing", "several", "another-kind"], + ) + async def test_anything_but_the_required_kind(self, result: Any, returned: str): + with pytest.raises( + AuthenticationError, match=f"returned a {returned} .*not a BearerToken" + ): + await refresh_if_stale(None, TOKEN_ONLY, Recording(result), now=NOW) + + async def test_an_expired_credential(self): + provider = Recording(token(-timedelta(minutes=1))) + with pytest.raises(AuthenticationError, match="expired"): + await refresh_if_stale(None, TOKEN_ONLY, provider, now=NOW) + + +# -------------------------------------------------------------------------- +# From a configuration to what to ask the provider for +# -------------------------------------------------------------------------- + +ARC: dict[str, Any] = {"kind": "arc", "endpoint": "https://arc.example.org:443"} +CE: dict[str, Any] = {"kind": "htcondor-ce", "endpoint": "ce.example.org:9619"} +SSH: dict[str, Any] = { + "kind": "batch", + "transport": "ssh", + "scheduler": "slurm", + "host": "batch.example.org", + "user": "dirac", + "known_hosts": "batch.example.org ssh-ed25519 AAAAC3Nz...", +} + + +class TestCredentialRequirements: + @pytest.mark.parametrize("base", [ARC, CE], ids=["arc", "htcondor-ce"]) + @pytest.mark.parametrize("auth", [{}, {"auth": "token"}], ids=["default", "named"]) + def test_a_ce_asks_for_a_token_for_its_endpoint( + self, base: dict[str, Any], auth: dict[str, Any] + ): + scopes = frozenset({"compute.create"}) + requirements = credential_requirements( + validate_config({**base, **auth}), scopes + ) + assert requirements is not None + assert requirements.kind is BearerToken + assert requirements.resource == base["endpoint"] + assert requirements.scopes == scopes + + @pytest.mark.parametrize( + ("auth", "kind"), [("key", SSHPrivateKey), ("password", SSHPassword)] + ) + def test_ssh_asks_for_the_login(self, auth: str, kind: type): + config = validate_config({**SSH, "auth": auth, "port": 2222}) + requirements = credential_requirements(config) + assert requirements is not None + assert requirements.kind is kind + assert requirements.resource == "dirac@batch.example.org:2222" + + @pytest.mark.parametrize( + "data", + [ + {**SSH, "auth": "agent"}, + {"kind": "batch", "transport": "local", "scheduler": "slurm"}, + ], + ids=["ssh-agent", "local"], + ) + def test_nothing_to_ask_for(self, data: dict[str, Any]): + assert credential_requirements(validate_config(data)) is None diff --git a/tests/unit/test_registry.py b/tests/unit/test_registry.py index eebab72..458d674 100644 --- a/tests/unit/test_registry.py +++ b/tests/unit/test_registry.py @@ -18,9 +18,11 @@ registry, ) from intercede._backends.config import FakeConfig +from intercede.auth import BearerToken, Credential, StaticCredentials from intercede.registry import _build, _load DUMMIES = "tests.unit._registry_dummies" +CREDENTIALS = StaticCredentials() CLOUD: dict[str, Any] = { "kind": "cloud", @@ -63,7 +65,7 @@ def composed(monkeypatch: pytest.MonkeyPatch) -> None: class TestBuilds: async def test_the_fake_for_the_test_suites(self): - async with _build(FakeConfig(resource="fake://test")) as backend: + async with _build(FakeConfig(resource="fake://test"), CREDENTIALS) as backend: assert isinstance(backend, JobBackend) assert backend.resource == "fake://test" @@ -72,29 +74,59 @@ async def test_the_fake_for_the_test_suites(self): [CLOUD, CloudConfig(**CLOUD)], ids=["mapping", "object"], ) - def test_a_single_class_backend_gets_its_settings( + def test_a_single_class_backend_gets_its_settings_and_provider( self, monkeypatch: pytest.MonkeyPatch, config: Any ): monkeypatch.setitem(registry._BACKENDS, "cloud", f"{DUMMIES}:DummyBackend") - built: Any = registry.get_backend(config) + provider = StaticCredentials() + built: Any = registry.get_backend(config, provider) assert type(built).__name__ == "DummyBackend" assert isinstance(built.settings, CloudConfig) + assert built.credentials is provider def test_a_batch_backend_is_composed_of_its_parts(self, composed: None): - built: Any = registry.get_backend({**SSH_SLURM, "queue": "short"}) + provider = StaticCredentials() + built: Any = registry.get_backend({**SSH_SLURM, "queue": "short"}, provider) assert type(built).__name__ == "DummyBatchBackend" - # Transport and scheduler each get the whole batch configuration. + # Transport and scheduler each get the whole batch configuration; the + # transport, which authenticates, also gets the provider. assert isinstance(built.transport.settings, BatchSSHConfig) assert built.transport.settings.host == "batch.example.org" + assert built.transport.credentials is provider assert built.scheduler.settings.queue == "short" + async def test_building_asks_the_provider_nothing( + self, composed: None, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setitem(registry._BACKENDS, "cloud", f"{DUMMIES}:DummyBackend") + + class Watching(StaticCredentials): + asked = 0 + + async def get(self, requirements: Any) -> Credential: + Watching.asked += 1 + return await super().get(requirements) + + registry.get_backend(SSH_SLURM, Watching()) + registry.get_backend(CLOUD, Watching()) + assert Watching.asked == 0 # construction does no I/O + class TestRefused: def test_invalid_configuration_keeps_the_details(self): with pytest.raises(ConfigurationError) as exc_info: - registry.get_backend({"kind": "arc"}) + registry.get_backend({"kind": "arc"}, CREDENTIALS) assert "endpoint: required for kind='arc'" in str(exc_info.value) + @pytest.mark.parametrize( + "credentials", + [None, BearerToken(value="t")], + ids=["none", "credential-not-provider"], + ) + def test_credentials_must_be_a_provider(self, composed: None, credentials: Any): + with pytest.raises(ConfigurationError, match="must be a CredentialProvider"): + registry.get_backend(LOCAL_SLURM, credentials) + @pytest.mark.parametrize( "config", [{"kind": "arc", "endpoint": "https://arc.example.org:443"}, CLOUD], @@ -104,18 +136,18 @@ def test_valid_but_not_shipped_yet(self, config: dict[str, Any]): with pytest.raises( RegistryError, match=f"backend '{config['kind']}' is not available" ): - registry.get_backend(config) + registry.get_backend(config, CREDENTIALS) def test_composed_part_not_shipped_yet(self): with pytest.raises(RegistryError, match="transport 'ssh' is not available"): - registry.get_backend(SSH_SLURM) + registry.get_backend(SSH_SLURM, CREDENTIALS) def test_an_entry_that_cannot_be_imported( self, composed: None, monkeypatch: pytest.MonkeyPatch ): monkeypatch.setitem(registry._TRANSPORTS, "ssh", "tests.unit.does_not_exist:X") with pytest.raises(RegistryError, match="cannot be imported") as exc_info: - registry.get_backend(SSH_SLURM) + registry.get_backend(SSH_SLURM, CREDENTIALS) assert isinstance(exc_info.value.__cause__, ImportError) @pytest.mark.parametrize( @@ -138,12 +170,16 @@ def test_what_breaks_its_protocol_is_refused( getattr(registry, table), name, f"{DUMMIES}:BuildsTheWrongThing" ) with pytest.raises(RegistryError, match=f"which is not a {protocol}"): - registry.get_backend(LOCAL_SLURM) + registry.get_backend(LOCAL_SLURM, CREDENTIALS) class TestAvailable: def test_shipped_today(self): - assert registry.available() == {"backend": (), "transport": (), "scheduler": ()} + assert registry.available() == { + "backend": ("htcondor-ce",), + "transport": (), + "scheduler": (), + } def test_lists_only_what_a_configuration_can_ask_for( self, composed: None, monkeypatch: pytest.MonkeyPatch @@ -170,5 +206,5 @@ def test_only_the_requested_entry_is_imported( monkeypatch.delitem(sys.modules, DUMMIES, raising=False) registry.available() assert DUMMIES not in sys.modules # listing imports nothing - registry.get_backend(LOCAL_SLURM) + registry.get_backend(LOCAL_SLURM, CREDENTIALS) assert DUMMIES in sys.modules