From 8e49e3051b66f71907f35fa975797cad2b89e49e Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 26 Oct 2026 14:59:06 -0700 Subject: [PATCH 1/5] docs: sync Funnel guides from develop --- .github/workflows/funnel-docs-check.yml | 29 +++ .github/workflows/sync-funnel-docs.yml | 77 ++++++ .gitignore | 2 + DEVELOPING.md | 208 ++++------------- README.md | 4 +- docs/tools/funnel/.funnel-docs.json | 43 ++++ docs/tools/funnel/docs/compute/grid-engine.md | 22 +- docs/tools/funnel/docs/compute/htcondor.md | 8 +- .../funnel/docs/compute/kubernetes-storage.md | 219 ++++++++++++++++++ docs/tools/funnel/docs/compute/pbs-torque.md | 11 +- docs/tools/funnel/docs/compute/slurm.md | 8 +- docs/tools/funnel/docs/databases/datastore.md | 2 +- docs/tools/funnel/docs/databases/postgres.md | 42 ++++ .../funnel/docs/development/developers.md | 6 +- docs/tools/funnel/docs/metrics/prometheus.md | 1 + docs/tools/funnel/docs/security/advanced.md | 3 +- docs/tools/funnel/docs/security/basic.md | 9 +- docs/tools/funnel/docs/storage/http.md | 42 +++- .../funnel/docs/storage/mounted-inputs.md | 114 +++++++++ docs/tools/funnel/index.md | 12 + scripts/sync_funnel_docs.py | 105 +++++++++ tests/test_sync_funnel_docs.py | 69 ++++++ zensical.toml | 29 ++- 23 files changed, 846 insertions(+), 219 deletions(-) create mode 100644 .github/workflows/funnel-docs-check.yml create mode 100644 .github/workflows/sync-funnel-docs.yml create mode 100644 docs/tools/funnel/.funnel-docs.json create mode 100644 docs/tools/funnel/docs/compute/kubernetes-storage.md create mode 100644 docs/tools/funnel/docs/databases/postgres.md create mode 100644 docs/tools/funnel/docs/storage/mounted-inputs.md create mode 100644 scripts/sync_funnel_docs.py create mode 100644 tests/test_sync_funnel_docs.py diff --git a/.github/workflows/funnel-docs-check.yml b/.github/workflows/funnel-docs-check.yml new file mode 100644 index 00000000..c828f5ae --- /dev/null +++ b/.github/workflows/funnel-docs-check.yml @@ -0,0 +1,29 @@ +name: Check Funnel documentation sync + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + name: Test sync and build site + runs-on: ubuntu-latest + steps: + - name: Check out CALYPR docs + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + - name: Test sync behavior + run: python -m unittest discover -s tests -v + - name: Install website dependencies + run: python -m pip install -r requirements.txt + - name: Build website + run: zensical build --clean diff --git a/.github/workflows/sync-funnel-docs.yml b/.github/workflows/sync-funnel-docs.yml new file mode 100644 index 00000000..52ad5239 --- /dev/null +++ b/.github/workflows/sync-funnel-docs.yml @@ -0,0 +1,77 @@ +name: Update Funnel documentation + +on: + schedule: + - cron: '17 * * * *' + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: sync-funnel-docs + cancel-in-progress: false + +jobs: + sync: + name: Prepare Funnel docs update + runs-on: ubuntu-latest + steps: + - name: Check out CALYPR docs + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check out Funnel develop + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: calypr/funnel + ref: develop + path: .funnel-upstream + persist-credentials: false + + - name: Check Funnel revision + id: revision + run: | + revision=$(git -C .funnel-upstream rev-parse HEAD) + previous=$(python3 -c 'import json; print(json.load(open("docs/tools/funnel/.funnel-docs.json"))["revision"])') + echo "revision=$revision" >> "$GITHUB_OUTPUT" + if [ "$revision" != "$previous" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Python + if: steps.revision.outputs.changed == 'true' + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Test sync behavior + if: steps.revision.outputs.changed == 'true' + run: python -m unittest discover -s tests -v + + - name: Sync Funnel documentation + if: steps.revision.outputs.changed == 'true' + run: python scripts/sync_funnel_docs.py .funnel-upstream --revision "${{ steps.revision.outputs.revision }}" + + - name: Install website dependencies + if: steps.revision.outputs.changed == 'true' + run: python -m pip install -r requirements.txt + + - name: Build the website + if: steps.revision.outputs.changed == 'true' + run: zensical build --clean + + - name: Create or update pull request + if: steps.revision.outputs.changed == 'true' + uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7 + with: + branch: automation/sync-funnel-docs + commit-message: 'docs: sync Funnel develop documentation' + title: 'docs: sync Funnel develop documentation' + body: | + Update the CALYPR site from [Funnel develop](https://github.com/calypr/funnel/tree/${{ steps.revision.outputs.revision }}) at `${{ steps.revision.outputs.revision }}`. + + Review the generated content and navigation before merging. This PR was prepared and built by the scheduled documentation sync. + delete-branch: true diff --git a/.gitignore b/.gitignore index f74ee97e..d9bc0c6e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ venv site/ .idea/ attic/ + +/.funnel-upstream/ diff --git a/DEVELOPING.md b/DEVELOPING.md index 5f9a3614..18866db1 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -1,176 +1,56 @@ # Developing CALYPR Docs -This repo now uses a Zensical-only build pipeline. MkDocs, `.nav.yml`, and the old multirepo plugin are gone. +The site is built with Zensical from the content in `docs/`. Navigation is +configured in `zensical.toml`. -The main thing to understand is that the published site is built from a generated docs tree, not directly from `docs/`. +## Build locally -## Mental Model - -- `docs/` contains repo-authored source content. -- `.generated/docs/` is the staged build input that Zensical actually reads. -- `site/` is the rendered output from `zensical build`. -- `zensical.toml` is the only navigation source of truth. - -The build pipeline is: - -1. Copy repo-authored docs from `docs/` into `.generated/docs/` -2. Import selected upstream docs from `git-drs` and `syfon` -3. Apply a few link rewrites needed for imported pages -4. Add generated assets like `termynal.css` and `termynal.js` -5. Run Zensical against `.generated/docs` - -## Core Commands - -From the repo root: - -```bash +```sh python3 -m venv venv source venv/bin/activate pip install -r requirements.txt -make serve +make build ``` -Useful commands: - -- `make prepare`: rebuild `.generated/docs` -- `make build`: run a clean Zensical build -- `make serve`: prepare docs and start the local server -- `make update-branches`: update GitHub source links in authored docs to the preferred upstream branches - -If you want the raw commands: - -```bash -python scripts/prepare_docs.py -zensical build --clean -zensical serve +Use `make serve` for local development. The rendered site is written to `site/`. +The [GitHub Pages workflow](.github/workflows/publish.yml) builds and publishes +when changes reach `main`. + +## Funnel documentation + +The canonical Funnel reference pages live in the `website/content/docs/` +directory on [Funnel's `develop` branch](https://github.com/calypr/funnel/tree/develop/website/content/docs). +A scheduled workflow checks that branch hourly and opens or updates a pull request +with changed pages and generated navigation. Review and merge the pull request +to publish the update. The workflow also runs the sync tests and builds the site. + +The Funnel landing page, download page, and storage index remain CALYPR site +content. The installation script is copied from Funnel's `install.sh`. The sync +converts Funnel's Hugo shortcodes to Markdown compatible with this site and +rejects unknown shortcodes rather than publishing them verbatim. + +Three pages are temporarily held at their existing CALYPR versions while newer +site guidance is reconciled with Funnel: the GCP Batch, Kubernetes, and S3 +guides. Their expected upstream versions are recorded in +`docs/tools/funnel/.funnel-docs.json`. Once those exact versions reach `develop`, +the sync imports them and removes their exceptions automatically. If the +upstream content changes during review, update the expected hashes after +reconciling the differences. + +To run the sync locally against a Funnel checkout: + +```sh +python scripts/sync_funnel_docs.py /path/to/funnel --revision "$(git -C /path/to/funnel rev-parse HEAD)" +python -m unittest discover -s tests -v +make build ``` -## Where Navigation Lives - -Navigation is fully defined in [zensical.toml](./zensical.toml). - -If you want to: - -- rename a sidebar label -- move a section -- add a new page to the nav -- change the Tools grouping - -edit the `nav = [...]` structure in `zensical.toml`. - -There are no `.nav.yml` files anymore. If you add a page under `docs/` and do not add it to `zensical.toml`, it will not appear in the intended nav structure. - -## Source of Truth Rules - -### Repo-authored pages - -These live in `docs/` and should be edited directly there. - -Examples: - -- `docs/index.md` -- `docs/calypr/...` -- `docs/tools/index.md` -- `docs/tools/funnel/...` -- `docs/tools/grip/...` - -### Imported tool pages - -These are not authored in this repo anymore. - -Current imported repos: - -- `git-drs` -- `syfon` - -Their imported file mappings live in [scripts/prepare_docs.py](./scripts/prepare_docs.py) under `REPO_IMPORTS`. - -If you need to: - -- add a new imported page -- stop importing a page -- change where an imported page lands under `tools/...` - -update `REPO_IMPORTS` and usually also update `zensical.toml`. - -Do not hand-edit imported pages under `.generated/docs`. They are build artifacts and will be replaced on the next `make prepare`. - -## How Upstream Imports Work - -The importer prefers local sibling repos by default: - -- `../git-drs` -- `../syfon` - -So on a typical local machine, the imported content comes from whatever is currently checked out in those repos. - -If local sibling repos are unavailable, the importer falls back to GitHub and picks the first available branch from [scripts/branch_config.json](./scripts/branch_config.json). - -Current branch preference order: - -- `development` -- `develop` -- `main` -- `master` - -The last import decision is written to `.generated/imports.json`. - -Environment overrides: - -- `CALYPR_IMPORT_PREFER_LOCAL=0`: force remote import mode -- `CALYPR_IMPORT_GIT_DRS_DIR=/path/to/git-drs`: override local `git-drs` source -- `CALYPR_IMPORT_SYFON_DIR=/path/to/syfon`: override local `syfon` source - -## When You Add or Change Pages - -### Adding a repo-authored page - -1. Create the file under `docs/` -2. Add it to `zensical.toml` -3. Run `make serve` or `make build` - -### Adding an imported tool page - -1. Add the file mapping in `scripts/prepare_docs.py` -2. Add the page to `zensical.toml` -3. If the imported markdown links break after relocation, add a targeted rewrite in `MARKDOWN_REWRITES` -4. Run `make prepare` -5. Run `make build` - -## Styling and Overrides - -Site-level assets still come from the repo: - -- custom CSS: `docs/stylesheets/extra.css` -- custom JS: `docs/javascripts/nav-open-level1.js` -- theme overrides: `overrides/` - -Those are copied into `.generated/docs` during preparation and then used by Zensical. - -## Deployment Behavior - -Deployment uses the same pipeline as local builds: - -- GitHub Pages runs `python scripts/prepare_docs.py` and then `zensical build --clean` -- Netlify does the same - -That means deployment now depends on the import step succeeding. In CI, the importer will usually use remote GitHub repos rather than local sibling repos. - -## Files You Should Usually Ignore - -- `.generated/` -- `site/` -- `.cache/` - -These are build/runtime artifacts. - -## Known Caveat - -Zensical currently reports some warnings from imported and legacy docs content, especially around bracketed literals like: - -- `[SIGNED]` -- `[0]` -- `[]string` -- `map[string]string` +Generated Funnel pages and the marked Funnel navigation block should be changed +in the Funnel repository, not edited manually here. The sync branch is reused +while an update PR is open. If the Funnel revision is unchanged, no PR is created. -Those are content/rendering warnings, not a sign that the Zensical migration is broken. +The workflow needs the repository setting that allows GitHub Actions to create +pull requests, plus `contents: write` and `pull-requests: write` job permissions. +GitHub places PR workflows created with `GITHUB_TOKEN` in an approval-required +state. The sync workflow runs its own tests and site build before creating the +PR; a repository writer can approve the PR workflows separately. diff --git a/README.md b/README.md index 9fc10980..d23c89d7 100644 --- a/README.md +++ b/README.md @@ -15,16 +15,14 @@ make serve The local build contract is: -- `make prepare` stages repo-authored docs plus imported upstream tool docs into `.generated/docs` - `make build` runs a clean Zensical build -- `make serve` stages the docs and starts the local Zensical server +- `make serve` starts the local Zensical server For repo maintenance details, navigation ownership, and import behavior, see [DEVELOPING.md](./DEVELOPING.md). If you prefer the raw commands: ```sh -python scripts/prepare_docs.py zensical serve ``` diff --git a/docs/tools/funnel/.funnel-docs.json b/docs/tools/funnel/.funnel-docs.json new file mode 100644 index 00000000..c6f72c86 --- /dev/null +++ b/docs/tools/funnel/.funnel-docs.json @@ -0,0 +1,43 @@ +{ + "source": "https://github.com/calypr/funnel", + "branch": "develop", + "revision": "a3be3b208e0f6202bc2a33d1946fb56a8e850431", + "files": [ + "compute/aws-batch.md", + "compute/deployment.md", + "compute/gcp-batch.md", + "compute/grid-engine.md", + "compute/htcondor.md", + "compute/kubernetes-storage.md", + "compute/kubernetes.md", + "compute/pbs-torque.md", + "compute/slurm.md", + "databases/boltdb.md", + "databases/datastore.md", + "databases/dynamodb.md", + "databases/elasticsearch.md", + "databases/mongodb.md", + "databases/postgres.md", + "development/developers.md", + "events/kafka.md", + "integrations/nextflow.md", + "integrations/py-tes.md", + "metrics/prometheus.md", + "security/advanced.md", + "security/basic.md", + "security/oauth2.md", + "storage/ftp.md", + "storage/google-storage.md", + "storage/http.md", + "storage/local.md", + "storage/mounted-inputs.md", + "storage/s3.md", + "storage/swift.md", + "tasks.md" + ], + "pending_reconciliation": { + "compute/gcp-batch.md": "a1ad1e72c4b7b8befd4ec98814855aa43fb48f8b76b6b12f9147ad64802f4d79", + "compute/kubernetes.md": "456c6cbed9d9d1687b717dde7e647daea896fd96b03662ec4022d862930ca400", + "storage/s3.md": "7624b293ea197bc68dfb01d06227b2c2da3b1ccd16762d2c00b3abfb01b4cf5b" + } +} diff --git a/docs/tools/funnel/docs/compute/grid-engine.md b/docs/tools/funnel/docs/compute/grid-engine.md index f885c55b..999e84e7 100644 --- a/docs/tools/funnel/docs/compute/grid-engine.md +++ b/docs/tools/funnel/docs/compute/grid-engine.md @@ -1,29 +1,27 @@ --- title: Grid Engine +menu: + main: + parent: Compute + weight: 20 --- # Grid Engine -Funnel can be configured to submit workers to [Grid Engine](https://gridscheduler.sourceforge.net/) by making calls +Funnel can be configured to submit workers to [Grid Engine][ge] by making calls to `qsub`. The Funnel server needs to run on a submission node. Configure Funnel to use Grid Engine by including the following config: It is recommended to update the submit file template so that the -`funnel worker run` command takes a config file as an argument: - -``` -{% raw %} -funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}} -{% endraw %} -``` +`funnel worker run` command takes a config file as an argument +(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`) ```YAML -{% raw %} Compute: gridengine GridEngine: - Template: | + Template: | #!/bin/bash #$ -N {{.TaskId}} #$ -o {{.WorkDir}}/funnel-stdout @@ -37,10 +35,9 @@ GridEngine: {{if ne .DiskGb 0.0 -}} {{printf "#$ -l h_fsize=%.0fG" .DiskGb}} {{- end}} + funnel worker run --taskID {{.TaskId}} -{% endraw %} ``` - The following variables are available for use in the template: | Variable | Description | @@ -54,3 +51,4 @@ The following variables are available for use in the template: See https://golang.org/pkg/text/template for information on creating templates. +[ge]: http://gridscheduler.sourceforge.net/documentation.html diff --git a/docs/tools/funnel/docs/compute/htcondor.md b/docs/tools/funnel/docs/compute/htcondor.md index a6c8ebdd..21c87873 100644 --- a/docs/tools/funnel/docs/compute/htcondor.md +++ b/docs/tools/funnel/docs/compute/htcondor.md @@ -14,15 +14,14 @@ The Funnel server needs to run on a submission node. Configure Funnel to use HTCondor by including the following config: It is recommended to update the submit file template so that the -`funnel worker run` command takes a config file as an argument {% raw %} -(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`){% endraw %} +`funnel worker run` command takes a config file as an argument +(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`) ```YAML -{% raw %} Compute: htcondor HTCondor: - Template: | + Template: | universe = vanilla getenv = True executable = funnel @@ -43,7 +42,6 @@ HTCondor: {{- end}} queue -{% endraw %} ``` The following variables are available for use in the template: diff --git a/docs/tools/funnel/docs/compute/kubernetes-storage.md b/docs/tools/funnel/docs/compute/kubernetes-storage.md new file mode 100644 index 00000000..e9c4fff9 --- /dev/null +++ b/docs/tools/funnel/docs/compute/kubernetes-storage.md @@ -0,0 +1,219 @@ +--- +title: Kubernetes storage +menu: + main: + parent: Compute + weight: 21 +--- + +# Kubernetes storage + +`Kubernetes.Storage` configures PVC-backed Kubernetes volumes for task workspaces and worker mounts. The `volumes` and `volumeMounts` fields use native Kubernetes objects, and configured volumes must use PVC sources. The `volumeClaimTemplates` and `persistentVolumeTemplates` fields contain native resource objects. `WorkspaceVolume` names the PVC-backed volume Funnel uses for the task workspace. `WorkerContainer` and `ExecutorContainer` select the containers that receive generated mounts when a Pod template has more than one regular container. + +When `Storage` is absent, Funnel keeps the existing `PVTemplate`, `PVCTemplate`, and Job template behavior. Explicit storage does not change TES inputs, outputs, or task paths. Configured GenericS3 mounted reads and writes require `Kubernetes.Storage` and a submitted storage plan. + +## Use an existing PVC + +Use `persistentVolumeClaim.claimName` for a claim that an operator already manages. Funnel does not create or delete that claim or its PV. + +```yaml +Worker: + WorkDir: /opt/funnel/funnel-work-dir +Kubernetes: + JobsNamespace: funnel-tasks + Storage: + WorkspaceVolume: workspace + volumes: + - name: workspace + persistentVolumeClaim: + claimName: shared-workspace +``` + +The claim must exist in `JobsNamespace` and support filesystem access from the worker and executor Pods. Funnel creates a task-specific workspace directory on that claim. `ReadWriteMany` supports placement across nodes. `ReadWriteOnce` requires compatible Pod placement. `ReadWriteOncePod` cannot serve the worker and executor Pods and is rejected. + +Use `volumes` entries for existing claims. Set `persistentVolumeClaim.claimName` to the claim's literal name. A PVC template creates a volume with the same logical name as the template. Do not add a matching `volumes` entry. A `volumeMounts` entry refers to either kind of volume through its `name` field. + +## Provision a PVC through a StorageClass + +Each PVC template creates a claim for one task and a volume with the same logical name. Set `WorkspaceVolume` to that name to use the claim for the task workspace. + +```yaml +Kubernetes: + JobsNamespace: funnel-tasks + Storage: + WorkspaceVolume: workspace + volumeClaimTemplates: + - metadata: + name: workspace + spec: + storageClassName: research-rwx + accessModes: + - ReadWriteMany + resources: + requests: + storage: "{{.DiskGb}}Gi" +``` + +For a task with ID `task-a`, this template creates the PVC `task-a-workspace` and a volume named `workspace`. Funnel sets the worker Job as the claim's owner. Kubernetes garbage collection removes the claim after the Job. The StorageClass determines provisioning and reclaim behavior. This configuration makes no PV API calls during normal submission or cleanup. + +Set each PVC template's `metadata.name` to a static logical volume name that is a DNS label, such as `workspace`. Funnel names the PVC `-`. Do not put `TaskId` in the PVC template's `metadata.name`. Omit `apiVersion` and `kind` to use `v1` and `PersistentVolumeClaim` defaults. Funnel infers an omitted `metadata.namespace` from the effective job namespace, which uses `JobsNamespace` or falls back to `Namespace`. Do not set `generateName`, owner references, or block volume mode. Each logical name must be unique and must not collide with a `volumes` entry. An existing-claim volume cannot refer to a generated claim name. + +Funnel renders Go templates in string values other than the PVC template's logical `metadata.name`, then decodes each object as a Kubernetes resource. It does not render an entire YAML document as template text. Quote templated quantity values so YAML keeps them as strings. Other fields can use `.TaskId`, `.Namespace`, and `.DiskGb`. They can also use `.Bucket`, `.Region`, and `.KmsKeyID` from the first GenericS3 entry when present. Those three values are empty when no GenericS3 entry exists. + +To migrate an existing task-specific PVC template, replace its `metadata.name` such as `work-{{.TaskId}}` with a static name such as `workspace`, then remove the `volumes` entry that points to the rendered claim name. If a PV template specifies `spec.claimRef.name`, remove that reference so Funnel fills the generated name, or change it to `{{.TaskId}}-workspace` for this example. Keep Go templates in other resource fields, including `spec.volumeName` and PV `metadata.name`. Existing version 4 storage plans remain compatible. The new naming rule applies to new task submissions. + +## Create an explicit PV and PVC pair + +Set `PVC.spec.volumeName` to the exact `metadata.name` of a `persistentVolumeTemplates` entry. The PVC template keeps its static logical name, while `spec.volumeName` can use `TaskId` to select a task-specific PV. Funnel fills a missing PV `spec.claimRef` with the generated PVC name and namespace. For task `task-a`, the example claim is `task-a-workspace`, and the PV claim reference uses that name. If the template provides a conflicting reference, Funnel rejects it before creating resources. + +```yaml +Kubernetes: + JobsNamespace: funnel-tasks + Storage: + WorkspaceVolume: workspace + volumeClaimTemplates: + - metadata: + name: workspace + spec: + storageClassName: "" + volumeName: disk-{{.TaskId}} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + persistentVolumeTemplates: + - apiVersion: v1 + kind: PersistentVolume + metadata: + name: disk-{{.TaskId}} + spec: + capacity: + storage: 20Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: "" + hostPath: + path: /srv/funnel/{{.TaskId}} +``` + +The `hostPath` source makes this example suitable only for a local cluster whose worker and executor Pods use the same node and path. Use the deployment's native CSI or other PV source in a real cluster. Each PV template must be a cluster-scoped `v1` `PersistentVolume` with an explicit `metadata.name` that includes `TaskId`. Do not set `metadata.namespace`, `generateName`, owner references, or block volume mode. A PV template must have exactly one native PV source and link to exactly one PVC template. + +Funnel creates the PV and PVC with task ownership and preserves the existing cleanup lifecycle. Explicit PV storage adds a finalizer to the worker Job. Cleanup requests deletion of owned PVs and the Job before releasing that finalizer. A failed delete request leaves the plan available for retry. Reconciliation retries cleanup for canceled tasks even when the Job is still running or its deletion request failed. Submission checks persisted cancellation before and after resource creation and cleans up resources created during cancellation. The server needs permission to delete these PVs. Finish PV-backed tasks and clean up their resources before removing cluster-scoped deletion permissions. Kubernetes storage-protection finalizers and the storage driver's normal cleanup still apply. + +Before creating the worker Job, Funnel checks PV permissions with Kubernetes `SelfSubjectAccessReview`. Explicit PV templates require cluster-scoped `create` permission and `get` and `delete` permission for each generated PV name. The check uses the effective task configuration, including plugin overrides. A denied or unavailable authorization check rejects submission before creating the Job or its storage. Existing-claim and StorageClass-only configurations skip these PV checks. + +The Funnel server identity must be allowed to create `selfsubjectaccessreviews` in the `authorization.k8s.io` API group. Kubernetes normally grants this through its default authenticated-user roles. The submission check does not require PV `list` permission; the separate orphan cleanup sweep still needs it. Authorization can change after the check, so keep PV permissions until all PV-backed tasks have been cleaned up. + +To enable explicit PV storage, an administrator can apply the following RBAC resources. Replace the subject name and namespace with the service account used by the Funnel server Pod. These permissions supplement the server's existing namespaced permissions for Jobs, Pods, and PVCs. + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: funnel-pv-manager +rules: + - apiGroups: [""] + resources: [persistentvolumes] + verbs: [create, get, delete] + - apiGroups: [authorization.k8s.io] + resources: [selfsubjectaccessreviews] + verbs: [create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: funnel-pv-manager +subjects: + - kind: ServiceAccount + name: funnel + namespace: funnel-tasks +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: funnel-pv-manager +``` + +## Add worker mounts + +`volumeMounts` contains native Kubernetes `volumeMount` objects. Use the configured volume's `name`. You can mount the same volume at multiple paths. + +```yaml +Kubernetes: + JobsNamespace: funnel-tasks + Storage: + WorkspaceVolume: workspace + volumes: + - name: workspace + persistentVolumeClaim: + claimName: shared-workspace + - name: reference-data + persistentVolumeClaim: + claimName: reference-datasets + volumeMounts: + - name: reference-data + mountPath: /mnt/reference + readOnly: true + subPath: genomes + - name: reference-data + mountPath: /mnt/annotations + readOnly: true + subPath: annotations +``` + +Funnel adds `volumeMounts` to the selected worker container. It generates the workspace mount for the worker and executor. Extra worker mounts do not expose data automatically at TES input paths. A storage backend must still prepare those inputs. Worker mount paths cannot overlap the task workspace, each other, or mounts already in the Job template. + +GenericS3 `MountedRead` and `MountedWrite` select a worker mount by volume name and, optionally, its static `subPath`. If one `volumeMounts` entry uses the selected volume, omit `SubPath`. If several entries use that volume, set `SubPath` to the exact `volumeMounts[].subPath` value. Set `SubPath: ""` to select a static root mount. A mount that uses `subPathExpr` can be selected only when it is the unique mount for that volume and `SubPath` is absent. + +The selected mount's authored `mountPath` is the S3 root. Funnel does not append the selected `SubPath` or the S3 `Prefix` to that path. A write mount must be writable. The underlying claim must also permit writes. + +## Configure Job templates + +With explicit storage, Funnel renders Job templates with `NeedsPVC: false` and adds resolved volumes and mounts afterward. Templates that already condition legacy storage on `NeedsPVC` keep their other settings. Remove template-defined volumes or mount paths that conflict with configured storage. + +Funnel selects the sole regular container by default. For templates with sidecars, set `Storage.WorkerContainer` and `Storage.ExecutorContainer` to the main container names. Funnel adds storage and worker configuration only to those containers. Init containers and sidecars keep their template-defined mounts. + +The worker Job name must equal `TaskId`. `Worker.WorkDir` must be absolute. Per-task plugin overrides cannot change `JobsNamespace`, because a backend reconciles one configured Jobs namespace. + +`Kubernetes.WorkerImage` selects the image supplied to the worker Job template and skips server Pod discovery. When it is absent, Funnel keeps the existing image discovery behavior. Templates use `{{.Image}}` to consume that setting. + +## Ownership and upgrades + +Funnel labels resources it creates and records the worker Job UID. Creation retries accept only resources with matching ownership and specifications. Funnel rejects an existing resource with a conflicting owner or specification instead of adopting it. Operator-owned claims remain under operator control. + +Provisioning checks the worker Job before and after each resource creation. If cancellation removes the Job during creation, Funnel deletes a late-created owned PV. A process crash, an ambiguous create response, or a failed compensating deletion can leave an orphan. Recovery requires an orphan cleanup sweep with PV scanning enabled by a PV template. An existing-claim-only configuration does not scan cluster-scoped PVs. + +Funnel records a version 4 plan in the Job's `funnel.calypr.org/storage-plan` annotation and supplies the same plan through `FUNNEL_KUBERNETES_STORAGE_PLAN`. The plan records resolved volumes, worker mounts, S3 routes, claim names, namespace, and work directory. It contains no credentials. Existing Jobs with version 4 plans remain compatible. Do not set the plan environment variable or annotation in templates. + +If the annotation is missing, cleanup recovers the plan from the Job's worker environment. It validates the task and namespace before using that copy. Conflicting copies, invalid plans, or a storage-finalized Job with neither copy cause an error and retain the finalizer. Restore the correct plan before retrying cleanup in those cases. + +Cleanup validates resource ownership separately from S3 transfer routes. Version 4 plans with write-route overlaps rejected by newer workers can still be cleaned up. New submissions and workers reject those overlaps. + +The broader native storage migration changes the config schema. Migrate configurations to native objects and rebuild Go RPC plugins against the updated config package before exchanging configuration with them. This PVC naming change adds no config fields and requires no plugin rebuild by itself. The schema preserves `Kubernetes.ForbiddenPathPrefixes` at field 20 and uses `WorkerImage` at 24 and `Storage` at 25. It reserves `KubernetesStorage` field 2 and the names `Bindings` and `MountedInputs`. `MountedRead` and `MountedWrite` reserve field 1 and the name `WorkerMount`. Kubernetes field numbers 21–23 and the names `PVCMode`, `StorageClassName`, `SharedPVCName`, and `S3FilesFilesystemId` are also reserved. Configuration with retired names fails parsing. + +For configurations from PR #1454, replace `PVCMode: full` with native PV and PVC templates, `PVCMode: pvc` with a PVC template containing `spec.storageClassName`, and `PVCMode: shared` with `persistentVolumeClaim.claimName`. Keep `WorkerImage` as an independent field. + +Chicago's `S3FilesFilesystemId` used field 20, which conflicts with upstream `ForbiddenPathPrefixes`. Do not decode old Chicago binary configuration with the upstream schema. Both fields use a length-delimited wire representation, so a decoder cannot distinguish them. Export the configuration with its original schema, move the filesystem ID into the deployment's native CSI PV template, and load the migrated named configuration. Rebuild plugins against the canonical schema before exchanging configuration. Funnel supplies no automatic decoder for the conflicting binary format. + +The version 4 decoder cannot run or clean up Jobs with version 1, 2, or 3 plans. Finish those tasks and clean up their Jobs with a compatible older binary before upgrading. Jobs without a storage plan still require the legacy cleanup configuration during migration. These requirements apply whether the task uses existing claims or native PVC templates. + +Helm resolves deployment values before Funnel renders task variables. Charts must preserve literal Funnel expressions such as `{{.TaskId}}` through Helm rendering, as they do for existing Funnel Job templates. + +## Verify against a local cluster + +The opt-in integration test uses an existing Kubernetes context, creates temporary namespaces, and runs worker and executor Jobs with `alpine:3.21`. Use a disposable single-node cluster with a `standard` StorageClass and access to that image. The test checks file exchange through a provisioned claim, an existing claim, and an explicit hostPath PV and PVC pair. It then verifies ownership-based cleanup, including PV protection and Job finalizers. + +```sh +FUNNEL_KUBERNETES_TEST_KUBECONFIG=/path/to/test-kubeconfig \ + go test ./compute/kubernetes -run '^TestNativeStorageLive$' -v -count=1 +``` + +The test verifies real Kubernetes mounts and cleanup. It does not certify an S3 CSI driver or asynchronous S3 output publication. + +To verify permission failures, run the RBAC test with an administrator kubeconfig for a disposable cluster. It creates temporary service accounts and roles, impersonates those accounts, and checks that denied PV permissions leave no worker Job or storage resources. It also checks successful submission and cleanup with PV permissions restricted by name and with a PVC-only configuration. + +```sh +FUNNEL_KUBERNETES_TEST_KUBECONFIG=/path/to/test-kubeconfig \ + go test ./compute/kubernetes -run '^TestNativeStoragePermissionsLive$' -v -count=1 +``` diff --git a/docs/tools/funnel/docs/compute/pbs-torque.md b/docs/tools/funnel/docs/compute/pbs-torque.md index 59b2dfa1..5a7751f5 100644 --- a/docs/tools/funnel/docs/compute/pbs-torque.md +++ b/docs/tools/funnel/docs/compute/pbs-torque.md @@ -1,6 +1,5 @@ --- title: PBS/Torque -render_macros: true menu: main: parent: Compute @@ -15,15 +14,14 @@ The Funnel server needs to run on a submission node. Configure Funnel to use PBS by including the following config: It is recommended to update the submit file template so that the -`funnel worker run` command takes a config file as an argument -(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {% raw %}{{.TaskId}}{% endraw %}`) +`funnel worker run` command takes a config file as an argument +(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`) -{% raw %} ```YAML Compute: pbs PBS: - Template: | + Template: | #!/bin/bash #PBS -N {{.TaskId}} #PBS -o {{.WorkDir}}/funnel-stdout @@ -40,7 +38,6 @@ PBS: funnel worker run --taskID {{.TaskId}} ``` -{% endraw %} The following variables are available for use in the template: | Variable | Description | @@ -54,4 +51,4 @@ The following variables are available for use in the template: See https://golang.org/pkg/text/template for information on creating templates. -[pbs]: https://hpc-wiki.info/hpc/Torque +[pbs]: http://www.adaptivecomputing.com/products/open-source/torque/ diff --git a/docs/tools/funnel/docs/compute/slurm.md b/docs/tools/funnel/docs/compute/slurm.md index 98f16977..6bb1243f 100644 --- a/docs/tools/funnel/docs/compute/slurm.md +++ b/docs/tools/funnel/docs/compute/slurm.md @@ -14,15 +14,14 @@ The Funnel server needs to run on a submission node. Configure Funnel to use Slurm by including the following config: It is recommended to update the submit file template so that the -`funnel worker run` command takes a config file as an argument -(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {% raw %}{{.TaskId}}{% endraw %}`) +`funnel worker run` command takes a config file as an argument +(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`) -{% raw %} ```YAML Compute: slurm Slurm: - Template: | + Template: | #!/bin/bash #SBATCH --job-name {{.TaskId}} #SBATCH --ntasks 1 @@ -40,7 +39,6 @@ Slurm: funnel worker run --taskID {{.TaskId}} ``` -{% endraw %} The following variables are available for use in the template: | Variable | Description | diff --git a/docs/tools/funnel/docs/databases/datastore.md b/docs/tools/funnel/docs/databases/datastore.md index ea31d8c6..9f39d1cd 100644 --- a/docs/tools/funnel/docs/databases/datastore.md +++ b/docs/tools/funnel/docs/databases/datastore.md @@ -38,7 +38,7 @@ to support the task-list queries. This is typically done through command-line by referencing an **index.yaml** file (do not change the filename) with the following content: -```shell +```sh gcloud datastore indexes create path/to/index.yaml --database='funnel' ``` diff --git a/docs/tools/funnel/docs/databases/postgres.md b/docs/tools/funnel/docs/databases/postgres.md new file mode 100644 index 00000000..b91c7952 --- /dev/null +++ b/docs/tools/funnel/docs/databases/postgres.md @@ -0,0 +1,42 @@ +--- +title: Postgres +menu: + main: + parent: Databases +--- + +# Postgres + +> [!WARNING] +> +> Postgres support is in early development. +> +> Production deployments are recommended to use the alternative storage backends available, such as MongoDB, ElasticSearch, DynamoDB, etc. + +## Configuration + +```yaml +Database: postgres + +Postgres: + Host: localhost + Database: funnel + User: funnel + Password: example +``` + +## Default Values + +```go + Postgres: &Postgres{ + Host: "localhost:5432", + Database: "funnel", + User: "funnel", + Password: "example", + Timeout: &TimeoutConfig{ + TimeoutOption: &TimeoutConfig_Duration{ + Duration: durationpb.New(time.Second * 30), + }, + }, + }, +``` diff --git a/docs/tools/funnel/docs/development/developers.md b/docs/tools/funnel/docs/development/developers.md index c18a4ec1..7769d901 100644 --- a/docs/tools/funnel/docs/development/developers.md +++ b/docs/tools/funnel/docs/development/developers.md @@ -17,7 +17,7 @@ This page contains a rough collection of notes for people wanting to build Funne 2. Ensure GOPATH is set. See [the docs][gopath] for help. Also, you probably want to add `$GOPATH/bin` to your `PATH`. 3. Clone funnel and build - ```shell + ```sh $ git clone https://github.com/ohsu-comp-bio/funnel.git $ cd funnel $ make @@ -36,7 +36,7 @@ A Funnel development environment includes: - [gRPC Gateway][gateway] for HTTP communication. - [Angular][angular] and [SASS][sass] for the web dashboard. - [GNU Make][make] for development tasks. -- [Docker](https://docker.io) for executing task containers (tested with v1.12, v1.13). +- [Docker][docker] for executing task containers (tested with v1.12, v1.13). - [dep][dep] for Go dependency vendoring. - [Make][make] for development/build commands. - [NodeJS][node] and [NPM][npm] for web dashboard development. @@ -64,6 +64,8 @@ mock interfaces in test code, for example, to mock the Google Cloud APIs. [grpc]: http://www.grpc.io/ [sass]: http://sass-lang.com/ [make]: https://www.gnu.org/software/make/ +[docker]: https://docker.io +[python]: https://www.python.org/ [dep]: https://golang.github.io/dep/ [node]: https://nodejs.org [npm]: https://www.npmjs.com/ diff --git a/docs/tools/funnel/docs/metrics/prometheus.md b/docs/tools/funnel/docs/metrics/prometheus.md index c59894b4..1b3495b6 100644 --- a/docs/tools/funnel/docs/metrics/prometheus.md +++ b/docs/tools/funnel/docs/metrics/prometheus.md @@ -32,4 +32,5 @@ Funnel exports these metrics: of bytes of disk space available by all nodes. [prom]: https://prometheus.io/ +[gauge]: https://prometheus.io/docs/concepts/metric_types/#gauge [graf]: https://grafana.com/ diff --git a/docs/tools/funnel/docs/security/advanced.md b/docs/tools/funnel/docs/security/advanced.md index 8812d759..3864e344 100644 --- a/docs/tools/funnel/docs/security/advanced.md +++ b/docs/tools/funnel/docs/security/advanced.md @@ -21,7 +21,8 @@ In this way this Wrapper can manage the bucket and the keys (the Wrapper would Stay tuned for this feature's development! This feature is being tracked with the following: -- Pull Request: https://github.com/calypr/funnel/pull/1098 +- GitHub Branch: https://github.com/ohsu-comp-bio/funnel/tree/feature/credentials +- Pull Request: https://github.com/ohsu-comp-bio/funnel/pull/1098 # Credits 🙌 diff --git a/docs/tools/funnel/docs/security/basic.md b/docs/tools/funnel/docs/security/basic.md index 0b19e071..7a2d9080 100644 --- a/docs/tools/funnel/docs/security/basic.md +++ b/docs/tools/funnel/docs/security/basic.md @@ -40,10 +40,15 @@ so you will also need to configure the RPC client. ```yaml RPCClient: - User: funnel - Password: abc123 + Credential: + User: funnel + Password: abc123 ``` +Beginning with Funnel v0.11.8, `User` and `Password` must be nested under +`RPCClient.Credential`. Configurations created for earlier releases need to be +updated to use this structure. + Make sure to properly protect the configuration file so that it's not readable by everyone: diff --git a/docs/tools/funnel/docs/storage/http.md b/docs/tools/funnel/docs/storage/http.md index 81922053..740d12b9 100644 --- a/docs/tools/funnel/docs/storage/http.md +++ b/docs/tools/funnel/docs/storage/http.md @@ -7,9 +7,36 @@ menu: # HTTP(S) -Funnel supports downloading files from public URLs via GET requests. No authentication -mechanism is allowed. This backend can be used to fetch objects from cloud storage -providers exposed using presigned URLs. +Funnel can download inputs from and upload outputs to HTTP(S) URLs that need no +credentials: public URLs, or presigned URLs issued by a cloud storage provider +such as Amazon S3 or Google Cloud Storage. No authentication mechanism is +allowed, so the URL itself must grant access. + +- Inputs are downloaded with a GET request. +- Outputs are uploaded with a PUT request that carries a `Content-Length` + header, as presigned S3 upload URLs require. + +Presigned URLs work for `FILE` inputs and outputs. A presigned URL is signed for +one exact object, but Funnel derives the URL of each file in a `DIRECTORY` +output, and of each match of a wildcard output, by appending the file's path to +the path of the output URL, keeping its query string. Those derived URLs carry +no valid signature, so use one `FILE` output per presigned URL. + +For `DIRECTORY` and wildcard outputs, point the output URL at a server that +signs each upload as it arrives, when the file name is known, and redirects it +to the storage provider: + +- An upload follows a 307 or 308 redirect and resends the file to the new URL. +- Uploads send `Expect: 100-continue` and wait up to one second for the server + to answer before sending the file. A server that redirects within that second + doesn't receive the file; a slower one may receive some or all of it first. +- A 301, 302 or 303 response to an upload is reported as an error, because + following it would turn the upload into a GET. +- Downloads follow redirects too. + +The URL is stored with the task, including any signature in its query string, +so issue presigned URLs with a short expiry. The storage provider limits the +size of a single PUT (5 GB on Amazon S3). The HTTP storage client is enabled by default, but may be explicitly disabled in the worker config: @@ -17,7 +44,7 @@ worker config: ```yaml HTTPStorage: Disabled: false - # Timeout for http(s) GET requests. + # Timeout for http(s) requests, including the transfer itself. Timeout: 30s ``` @@ -29,9 +56,14 @@ HTTPStorage: "url": "http://fakedomain.com/hello.txt", "path": "/inputs/hello.txt" }], + "outputs": [{ + "url": "https://my-bucket.s3.amazonaws.com/hello-upper.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=...", + "path": "/outputs/hello-upper.txt", + "type": "FILE" + }], "executors": [{ "image": "alpine", - "command": ["cat", "/inputs/hello.txt"], + "command": ["sh", "-c", "tr a-z A-Z < /inputs/hello.txt > /outputs/hello-upper.txt"] }] } ``` diff --git a/docs/tools/funnel/docs/storage/mounted-inputs.md b/docs/tools/funnel/docs/storage/mounted-inputs.md new file mode 100644 index 00000000..e8b665cb --- /dev/null +++ b/docs/tools/funnel/docs/storage/mounted-inputs.md @@ -0,0 +1,114 @@ +--- +title: Mounted inputs +menu: + main: + parent: Storage +--- + +# Mounted inputs + +`GenericS3.MountedRead` and `GenericS3.MountedWrite` let a Kubernetes worker read inputs or publish outputs through PVC-backed filesystem mounts. Each setting names a configured storage volume. If that volume has multiple worker mounts, use `SubPath` to select the mount whose static Kubernetes `subPath` matches exactly. Read and write routes are optional and can use separate volumes or mount views. + +```yaml +GenericS3: + - Disabled: false + Endpoint: https://s3.example.org + Bucket: analysis-data + Key: storage-key + Secret: storage-secret + Region: us-east-1 + MountedRead: + Volume: reference-data + SubPath: genomes + Prefix: genomes/ + MountedWrite: + Volume: result-data + Prefix: results/ + Timeout: 2700s + +Kubernetes: + JobsNamespace: funnel-tasks + Storage: + WorkspaceVolume: workspace + volumes: + - name: workspace + persistentVolumeClaim: + claimName: task-workspace + - name: reference-data + persistentVolumeClaim: + claimName: reference-pvc + - name: result-data + persistentVolumeClaim: + claimName: task-results + volumeMounts: + - name: reference-data + mountPath: /mnt/reference + readOnly: true + subPath: genomes + - name: reference-data + mountPath: /mnt/annotations + readOnly: true + subPath: annotations + - name: result-data + mountPath: /mnt/results +``` + +The two reference mounts use one PVC at different paths. `MountedRead.SubPath: genomes` selects the first view. `MountedWrite` omits `SubPath` because `result-data` has one worker mount. If a volume has several mounts, set `SubPath` to the exact static `volumeMounts[].subPath` value. Set `SubPath: ""` to select a static root mount. A mount with `subPathExpr` can be selected only when it is the unique mount for that volume and `SubPath` is absent. + +`Prefix` is relative to the bucket. An empty prefix selects the whole bucket. The selected mount's authored `mountPath` is the route root. For example, `s3://analysis-data/genomes/genome.fa` maps to `/mnt/reference/genome.fa`. Funnel does not append the selected subpath or prefix to `mountPath`. The storage driver and operator must make the mounted view correspond to the configured bucket prefix and provide the required freshness. + +`MountedWrite` uses the same volume, subpath, and prefix rules. Its selected mount must be writable, and the underlying PVC must permit writes. Before creating task resources, Funnel checks both existing claims and PVC templates for `ReadWriteOnce`, `ReadWriteMany`, or `ReadWriteOncePod` access. A claim with only `ReadOnlyMany` cannot serve mounted outputs. The workspace still cannot use `ReadWriteOncePod` because worker and executor Pods share it. Funnel appends an output key path relative to the configured prefix when it copies a TES output file into the mount. + +For native storage configuration details, including existing claims, StorageClass-provisioned claims, and explicit PV/PVC pairs, see [Kubernetes storage](../../compute/kubernetes-storage/). + +## TES behavior + +Clients continue to submit ordinary TES inputs. They do not supply PVC names, volume names, worker mount paths, or deployment paths. + +```json +{ + "inputs": [ + { + "url": "s3://analysis-data/genomes/genome.fa", + "path": "/inputs/genome.fa", + "type": "FILE" + } + ], + "executors": [ + { + "image": "alpine:3.21", + "command": ["cat", "/inputs/genome.fa"] + } + ] +} +``` + +The worker copies each input into the task workspace. For a matching GenericS3 mounted read, it reads the source from the selected filesystem mount. The executor sees the copy at `/inputs/genome.fa`. Changing that copy does not change the mounted source. Directory inputs preserve their relative filenames. Reference mounts remain worker-only. They do not change TES `volumes`, which retain their empty scratch-directory behavior. + +## Read routing and safety + +The worker activates a mounted read or write only when the submitted storage plan names the exact GenericS3 `Endpoint` and `Bucket`. The plan stores `S3ReadMounts` and `S3WriteMounts` with the endpoint, bucket, prefix, and mount root, without credentials. The worker ignores static `MountedRead` and `MountedWrite` settings, including when the submitted plan has no mounted routes. Configured mounted reads and writes require `Kubernetes.Storage` and its submitted plan. The API server's `NewMux` does not activate filesystem reads or writes. + +GenericS3 handles inputs whose bucket-relative key matches its configured prefix. Funnel treats S3 keys literally. Spaces, `#`, and percent signs remain part of the key. For example, `a%20b.txt` names the literal file `a%20b.txt`. Prefix matching uses whole path components, so `genomes/` does not match `genomes-old/file.fa`. Unmatched URLs use the configured storage API path. + +A matched file must exist beneath the selected mount root. Funnel rejects traversal, encoded traversal forms, and escaping symlinks through rooted filesystem operations. Regular-file symlinks within the root can be read. Directory listings reject directory symlinks and special files. A missing or unsafe matched input fails the transfer. Funnel does not fall back to the API for that input. + +## Configuration ownership + +The server resolves each backend's selected worker volume mount and records mounted reads and writes in version 4 storage plans. Workers use the plan directly, even when their static ConfigMap differs. The server does not read worker filesystem paths. Existing Jobs with version 4 plans remain compatible. The version 4 decoder cannot run or clean up Jobs with version 1, 2, or 3 plans. Finish those tasks and clean up their Jobs with a compatible older binary before upgrading. + +The broader native storage migration changes the config schema. Replace `MountedRead.WorkerMount` and `MountedWrite.WorkerMount` with `Volume`, and set `SubPath` when one volume has multiple explicit worker mounts. Rebuild Go RPC plugins against the updated config package before exchanging configuration with them. The automatic PVC naming change does not alter the plugin schema or require another rebuild. The schema reserves field 1 and the name `WorkerMount` in `MountedRead` and `MountedWrite`. Other compute backends retain their existing storage clients. No TES schema fields change. + +## Output publication + +For an output URL that matches a configured `MountedWrite` endpoint, bucket, and prefix, the worker copies each output file to a private temporary file beside its destination. After closing the file, it renames the temporary file over the destination. Concurrent transfers publish complete files instead of copying into the same inode. If the source already is the mounted destination, Funnel skips the copy. It then calls S3 `HeadObject` until the object exists with the same size as the local file. The worker checks zero-byte files too. + +Mounted file transfers require Unix. The filesystem must support exclusive temporary-file creation, POSIX file permissions, and atomic replacement through same-directory rename, including across worker Pods. A failed rename fails the transfer. Funnel does not retry through an in-place copy or an API upload. Separate write routes must use nonoverlapping filesystem views. Funnel rejects overlapping roots and overlapping views of the same PVC that it can identify from the storage plan. + +After a failed or canceled copy, Funnel attempts to remove its temporary file and reports cleanup errors. A process crash can leave temporary files, and a storage driver can export temporary names to the bucket before Funnel removes them. Account for these files in the deployment's storage cleanup policy. Filesystem replacement does not make the driver's later S3 synchronization atomic. See [S3 Files synchronization](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-files-synchronization.html) for that driver's rename and export behavior. + +This check proves that S3 reports an object with the expected size. It does not compare file contents, checksums, or object versions. A same-sized object left by an earlier task can satisfy the check. Set `MountedWrite.Timeout` to bound polling. The example's `2700s` timeout is 45 minutes. The default is three hours, and the worker polls every two seconds. An explicit timeout must be positive. Task cancellation also stops the wait. + +The GenericS3 credentials must allow `HeadObject` for mounted destinations. They do not need upload permission for an output handled by the mount. An output URL that does not match the configured endpoint, bucket, and prefix follows the regular GenericS3 API upload path, which requires upload permission. Outputs do not use a `MountedRead` setting. + +Funnel handles directory outputs as individual files. It records successful files in TES output logs and preserves partial successes when another file fails. A timeout, cancellation, or transfer error prevents successful task completion. Retries retain one successful log entry per file after publication succeeds. diff --git a/docs/tools/funnel/index.md b/docs/tools/funnel/index.md index 05176f5e..800aeca1 100644 --- a/docs/tools/funnel/index.md +++ b/docs/tools/funnel/index.md @@ -8,6 +8,18 @@ Funnel provides task execution and workflow orchestration around the GA4GH TES m It is designed to streamline distributed batch processing by exposing a simple task API and a set of components that can be adapted to different compute and storage backends. +## Connect to a remote server + +Use `-S` to point a command at a Funnel server, or set `FUNNEL_SERVER` for your shell: + +```sh +funnel -S https://funnel.example.com task list +export FUNNEL_SERVER=https://funnel.example.com +funnel task list +``` + +The `-S` flag takes precedence over `FUNNEL_SERVER` when both are set. + ## Read Next - [Architecture Overview](docs/docs.md) diff --git a/scripts/sync_funnel_docs.py b/scripts/sync_funnel_docs.py new file mode 100644 index 00000000..6276594a --- /dev/null +++ b/scripts/sync_funnel_docs.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Copy Funnel's reviewed develop docs into the CALYPR website checkout.""" +import argparse +import hashlib +import json +import re +from pathlib import Path + +SITE = Path(__file__).resolve().parents[1] +DEST = SITE / 'docs/tools/funnel/docs' +MANIFEST = SITE / 'docs/tools/funnel/.funnel-docs.json' +NAV = SITE / 'zensical.toml' +GROUPS = [('Storage', 'storage'), ('Compute', 'compute'), ('Databases', 'databases'), + ('Security', 'security'), ('Integrations', 'integrations'), ('Events', 'events'), + ('Metrics', 'metrics'), ('Development', 'development')] +TEMPLATES = {'htcondor': ('htcondor', 'HTCondor'), 'gridengine': ('gridengine', 'GridEngine'), + 'pbs': ('pbs', 'PBS'), 'slurm': ('slurm', 'Slurm')} + +def convert(text, upstream): + text = text.replace('https://ohsu-comp-bio.github.io/funnel/install.sh', 'https://calypr.org/tools/funnel/install.sh') + text = text.replace('src="/img/k8s-pvc.png"', 'src="../k8s-pvc.png"') + text = re.sub(r'\{\{< details title="([^"]*)" >\}\}', r'
\1\n', text) + text = text.replace('{{< /details >}}', '
') + converted_template = False + for shortcode, (compute, config) in TEMPLATES.items(): + token = '{{< ' + shortcode + '-template >}}' + if token in text: + converted_template = True + template = (upstream / 'website/static/funnel-config-examples' / (shortcode + '-template.txt')).read_text() + replacement = 'Compute: ' + compute + '\n\n' + config + ':\n Template: |\n' + ''.join(' ' + line + '\n' for line in template.splitlines()) + text = text.replace(token, replacement.rstrip()) + if re.search(r'\{\{[<%]', text): + raise ValueError('Unconverted Hugo shortcode; extend the sync before importing this page') + if converted_template: + text = re.sub(r'(?m)^[ \t]+$', '', text) + text = text.replace('as an argument \n', 'as an argument\n') + return text + +def title(path): + match = re.search(r'^title:\s*["\']?(.+?)["\']?\s*$', path.read_text(), re.MULTILINE) + return match.group(1) if match else path.stem.replace('-', ' ').title() + +def sync(upstream, revision): + source = upstream / 'website/content/docs' + files = [p for p in source.rglob('*.md') if p.parent != source or p.name == 'tasks.md'] + unknown = {p.relative_to(source).parts[0] for p in files if p.parent != source} - {folder for _, folder in GROUPS} + if unknown: + raise ValueError('Unknown Funnel documentation sections: ' + ', '.join(sorted(unknown))) + if not files or not (source / 'tasks.md').is_file(): + raise ValueError('Not a Funnel source checkout: website/content/docs/tasks.md is missing') + old = json.loads(MANIFEST.read_text()) if MANIFEST.exists() else {'files': []} + pending = old.get('pending_reconciliation', {}).copy() + new = sorted(p.relative_to(source).as_posix() for p in files) + for name in old['files']: + p = Path(name) + if p.is_absolute() or '..' in p.parts: + raise ValueError('Invalid path in sync manifest') + if name not in new: + (DEST / p).unlink(missing_ok=True) + for path in files: + name = path.relative_to(source).as_posix() + if name in pending: + if hashlib.sha256(path.read_bytes()).hexdigest() != pending[name]: + continue + del pending[name] + target = DEST / path.relative_to(source) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(convert(path.read_text(), upstream)) + image = upstream / 'website/static/img/k8s-pvc.png' + if image.is_file(): + (DEST / 'compute/k8s-pvc.png').write_bytes(image.read_bytes()) + # This public installer is also distributed directly by the CALYPR site. + (SITE / 'docs/tools/funnel/install.sh').write_bytes((upstream / 'install.sh').read_bytes()) + + nav = [' # BEGIN GENERATED FUNNEL NAV', ' { "Funnel" = [', + ' "tools/funnel/index.md",', ' { "Overview" = "tools/funnel/docs/docs.md" },', + ' { "Download" = "tools/funnel/docs/download.md" },', + ' { "Tasks" = "tools/funnel/docs/tasks.md" },'] + for label, folder in GROUPS: + entries = sorted((p for p in files if p.relative_to(source).parts[0] == folder), key=lambda p: (title(p).lower(), p.name)) + if not entries: + continue + nav.append(' { ' + json.dumps(label) + ' = [') + if folder == 'storage': + nav.append(' "tools/funnel/docs/storage/index.md",') + for p in entries: + nav.append(' { ' + json.dumps(title(p), ensure_ascii=False) + ' = ' + json.dumps('tools/funnel/docs/' + p.relative_to(source).as_posix()) + ' },') + nav.append(' ] },') + nav += [' ] },', ' # END GENERATED FUNNEL NAV'] + config = NAV.read_text() + begin, end = ' # BEGIN GENERATED FUNNEL NAV', ' # END GENERATED FUNNEL NAV' + if begin not in config or end not in config: + raise ValueError('Missing Funnel navigation markers') + NAV.write_text(config[:config.index(begin)] + '\n'.join(nav) + config[config.index(end)+len(end):]) + MANIFEST.write_text(json.dumps({'source': 'https://github.com/calypr/funnel', 'branch': 'develop', 'revision': revision, 'files': new, 'pending_reconciliation': pending}, indent=2) + '\n') + print(f'Synced {len(new)} Funnel pages from {revision}') + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('upstream', type=Path) + parser.add_argument('--revision', required=True) + args = parser.parse_args() + if not re.fullmatch(r'[a-f0-9]{40}', args.revision): + parser.error('--revision must be a full Git commit SHA') + sync(args.upstream.resolve(), args.revision) diff --git a/tests/test_sync_funnel_docs.py b/tests/test_sync_funnel_docs.py new file mode 100644 index 00000000..f80d6488 --- /dev/null +++ b/tests/test_sync_funnel_docs.py @@ -0,0 +1,69 @@ +import hashlib +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location('sync_funnel_docs', Path(__file__).resolve().parents[1] / 'scripts/sync_funnel_docs.py') +sync = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(sync) +REV = 'a' * 40 + +class SyncTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + root = Path(self.temp.name) + self.site, self.upstream = root / 'site', root / 'upstream' + self.source = self.upstream / 'website/content/docs' + (self.source / 'compute').mkdir(parents=True) + (self.site / 'docs/tools/funnel/docs').mkdir(parents=True) + (self.upstream / 'install.sh').write_text('#!/bin/sh\n') + (self.source / 'tasks.md').write_text('---\ntitle: Tasks\n---\n') + (self.source / 'compute/example.md').write_text('---\ntitle: Example\n---\nnew\n') + self.nav = self.site / 'zensical.toml' + self.nav.write_text(' # BEGIN GENERATED FUNNEL NAV\n # END GENERATED FUNNEL NAV\n') + sync.SITE = self.site + sync.DEST = self.site / 'docs/tools/funnel/docs' + sync.MANIFEST = self.site / 'docs/tools/funnel/.funnel-docs.json' + sync.NAV = self.nav + + def tearDown(self): + self.temp.cleanup() + + def test_sync_is_repeatable_and_updates_navigation_and_removed_pages(self): + sync.sync(self.upstream, REV) + page = sync.DEST / 'compute/example.md' + self.assertIn('new', page.read_text()) + self.assertIn('tools/funnel/docs/compute/example.md', self.nav.read_text()) + old_nav, old_manifest = self.nav.read_text(), sync.MANIFEST.read_text() + sync.sync(self.upstream, REV) + self.assertEqual(old_nav, self.nav.read_text()) + self.assertEqual(old_manifest, sync.MANIFEST.read_text()) + (self.source / 'compute/example.md').unlink() + sync.sync(self.upstream, REV) + self.assertFalse(page.exists()) + self.assertNotIn('tools/funnel/docs/compute/example.md', self.nav.read_text()) + + def test_preserves_live_page_until_reconciled_upstream_arrives(self): + page = sync.DEST / 'compute/example.md' + page.parent.mkdir(parents=True) + page.write_text('live content') + expected = b'---\ntitle: Example\n---\nreconciled\n' + sync.MANIFEST.write_text(json.dumps({'files': [], 'pending_reconciliation': {'compute/example.md': hashlib.sha256(expected).hexdigest()}})) + sync.sync(self.upstream, REV) + self.assertEqual(page.read_text(), 'live content') + (self.source / 'compute/example.md').write_bytes(expected) + sync.sync(self.upstream, REV) + self.assertEqual(page.read_bytes(), expected) + self.assertEqual(json.loads(sync.MANIFEST.read_text())['pending_reconciliation'], {}) + (self.source / 'compute/example.md').write_text('later change') + sync.sync(self.upstream, REV) + self.assertEqual(page.read_text(), 'later change') + + def test_fails_for_unknown_shortcodes(self): + with self.assertRaisesRegex(ValueError, 'Unconverted Hugo shortcode'): + sync.convert('{{< surprise >}}', self.upstream) + +if __name__ == '__main__': + unittest.main() diff --git a/zensical.toml b/zensical.toml index 4651955e..004b97fa 100644 --- a/zensical.toml +++ b/zensical.toml @@ -93,6 +93,7 @@ nav = [ ] }, ] }, { "Workflow Execution" = [ + # BEGIN GENERATED FUNNEL NAV { "Funnel" = [ "tools/funnel/index.md", { "Overview" = "tools/funnel/docs/docs.md" }, @@ -100,34 +101,37 @@ nav = [ { "Tasks" = "tools/funnel/docs/tasks.md" }, { "Storage" = [ "tools/funnel/docs/storage/index.md", - { "Local" = "tools/funnel/docs/storage/local.md" }, - { "S3" = "tools/funnel/docs/storage/s3.md" }, + { "FTP" = "tools/funnel/docs/storage/ftp.md" }, { "Google Storage" = "tools/funnel/docs/storage/google-storage.md" }, - { "OpenStack Swift" = "tools/funnel/docs/storage/swift.md" }, { "HTTP(S)" = "tools/funnel/docs/storage/http.md" }, - { "FTP" = "tools/funnel/docs/storage/ftp.md" }, + { "Local" = "tools/funnel/docs/storage/local.md" }, + { "Mounted inputs" = "tools/funnel/docs/storage/mounted-inputs.md" }, + { "OpenStack Swift" = "tools/funnel/docs/storage/swift.md" }, + { "S3" = "tools/funnel/docs/storage/s3.md" }, ] }, { "Compute" = [ - { "Deploying a Cluster" = "tools/funnel/docs/compute/deployment.md" }, { "AWS Batch" = "tools/funnel/docs/compute/aws-batch.md" }, + { "Deploying a cluster" = "tools/funnel/docs/compute/deployment.md" }, { "GCP Batch" = "tools/funnel/docs/compute/gcp-batch.md" }, - { "Kubernetes" = "tools/funnel/docs/compute/kubernetes.md" }, + { "Grid Engine" = "tools/funnel/docs/compute/grid-engine.md" }, { "HTCondor" = "tools/funnel/docs/compute/htcondor.md" }, - { "Slurm" = "tools/funnel/docs/compute/slurm.md" }, + { "Kubernetes" = "tools/funnel/docs/compute/kubernetes.md" }, + { "Kubernetes storage" = "tools/funnel/docs/compute/kubernetes-storage.md" }, { "PBS/Torque" = "tools/funnel/docs/compute/pbs-torque.md" }, - { "Grid Engine" = "tools/funnel/docs/compute/grid-engine.md" }, + { "Slurm" = "tools/funnel/docs/compute/slurm.md" }, ] }, { "Databases" = [ + { "Datastore" = "tools/funnel/docs/databases/datastore.md" }, + { "DynamoDB" = "tools/funnel/docs/databases/dynamodb.md" }, + { "Elasticsearch" = "tools/funnel/docs/databases/elasticsearch.md" }, { "Embedded" = "tools/funnel/docs/databases/boltdb.md" }, { "MongoDB" = "tools/funnel/docs/databases/mongodb.md" }, - { "Elasticsearch" = "tools/funnel/docs/databases/elasticsearch.md" }, - { "DynamoDB" = "tools/funnel/docs/databases/dynamodb.md" }, - { "Datastore" = "tools/funnel/docs/databases/datastore.md" }, + { "Postgres" = "tools/funnel/docs/databases/postgres.md" }, ] }, { "Security" = [ + { "Advanced Auth" = "tools/funnel/docs/security/advanced.md" }, { "Basic Auth" = "tools/funnel/docs/security/basic.md" }, { "OAuth2" = "tools/funnel/docs/security/oauth2.md" }, - { "Advanced Auth" = "tools/funnel/docs/security/advanced.md" }, ] }, { "Integrations" = [ { "Nextflow" = "tools/funnel/docs/integrations/nextflow.md" }, @@ -143,6 +147,7 @@ nav = [ { "Funnel Developers" = "tools/funnel/docs/development/developers.md" }, ] }, ] }, + # END GENERATED FUNNEL NAV ] }, { "Graph Database" = [ { "GRIP" = [ From a4f8dbfc3f6de7bcfa0e3e4cd341dab8907c6062 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 26 Oct 2026 15:00:44 -0700 Subject: [PATCH 2/5] docs: clarify sync organization permissions --- DEVELOPING.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DEVELOPING.md b/DEVELOPING.md index 18866db1..62a05f79 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -49,8 +49,9 @@ Generated Funnel pages and the marked Funnel navigation block should be changed in the Funnel repository, not edited manually here. The sync branch is reused while an update PR is open. If the Funnel revision is unchanged, no PR is created. -The workflow needs the repository setting that allows GitHub Actions to create -pull requests, plus `contents: write` and `pull-requests: write` job permissions. +The workflow needs both the organization policy and the repository setting that +allow GitHub Actions to create pull requests, plus `contents: write` and +`pull-requests: write` job permissions. GitHub places PR workflows created with `GITHUB_TOKEN` in an approval-required state. The sync workflow runs its own tests and site build before creating the PR; a repository writer can approve the PR workflows separately. From 91c44f1a75a0501a9241e0a0f767b9463e2b50a3 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 26 Oct 2026 15:01:56 -0700 Subject: [PATCH 3/5] fix: protect pending docs during sync --- scripts/sync_funnel_docs.py | 3 +++ tests/test_sync_funnel_docs.py | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/scripts/sync_funnel_docs.py b/scripts/sync_funnel_docs.py index 6276594a..d8f5c981 100644 --- a/scripts/sync_funnel_docs.py +++ b/scripts/sync_funnel_docs.py @@ -51,6 +51,9 @@ def sync(upstream, revision): old = json.loads(MANIFEST.read_text()) if MANIFEST.exists() else {'files': []} pending = old.get('pending_reconciliation', {}).copy() new = sorted(p.relative_to(source).as_posix() for p in files) + missing_pending = set(pending) - set(new) + if missing_pending: + raise ValueError('Reconciliation pages missing upstream: ' + ', '.join(sorted(missing_pending))) for name in old['files']: p = Path(name) if p.is_absolute() or '..' in p.parts: diff --git a/tests/test_sync_funnel_docs.py b/tests/test_sync_funnel_docs.py index f80d6488..cd36c600 100644 --- a/tests/test_sync_funnel_docs.py +++ b/tests/test_sync_funnel_docs.py @@ -61,6 +61,16 @@ def test_preserves_live_page_until_reconciled_upstream_arrives(self): sync.sync(self.upstream, REV) self.assertEqual(page.read_text(), 'later change') + def test_does_not_delete_a_pending_page_when_upstream_removes_it(self): + page = sync.DEST / 'compute/example.md' + page.parent.mkdir(parents=True) + page.write_text('live content') + sync.MANIFEST.write_text(json.dumps({'files': ['compute/example.md'], 'pending_reconciliation': {'compute/example.md': 'expected'}})) + (self.source / 'compute/example.md').unlink() + with self.assertRaisesRegex(ValueError, 'Reconciliation pages missing upstream'): + sync.sync(self.upstream, REV) + self.assertEqual(page.read_text(), 'live content') + def test_fails_for_unknown_shortcodes(self): with self.assertRaisesRegex(ValueError, 'Unconverted Hugo shortcode'): sync.convert('{{< surprise >}}', self.upstream) From af6a56249bbf38d72174d590531a2487ebec6c20 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 26 Oct 2026 15:05:11 -0700 Subject: [PATCH 4/5] ci: check Funnel docs updates daily --- .github/workflows/sync-funnel-docs.yml | 2 +- DEVELOPING.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-funnel-docs.yml b/.github/workflows/sync-funnel-docs.yml index 52ad5239..147aaa76 100644 --- a/.github/workflows/sync-funnel-docs.yml +++ b/.github/workflows/sync-funnel-docs.yml @@ -2,7 +2,7 @@ name: Update Funnel documentation on: schedule: - - cron: '17 * * * *' + - cron: '17 6 * * *' workflow_dispatch: permissions: diff --git a/DEVELOPING.md b/DEVELOPING.md index 62a05f79..475eadde 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -20,7 +20,7 @@ when changes reach `main`. The canonical Funnel reference pages live in the `website/content/docs/` directory on [Funnel's `develop` branch](https://github.com/calypr/funnel/tree/develop/website/content/docs). -A scheduled workflow checks that branch hourly and opens or updates a pull request +A scheduled workflow checks that branch daily and opens or updates a pull request with changed pages and generated navigation. Review and merge the pull request to publish the update. The workflow also runs the sync tests and builds the site. From 1729ee0d8f635a473758ffcdba50d73b6ec86c3f Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 26 Oct 2026 15:08:47 -0700 Subject: [PATCH 5/5] chore: leave Funnel content to scheduled sync --- docs/tools/funnel/.funnel-docs.json | 34 +-- docs/tools/funnel/docs/compute/grid-engine.md | 22 +- docs/tools/funnel/docs/compute/htcondor.md | 8 +- .../funnel/docs/compute/kubernetes-storage.md | 219 ------------------ docs/tools/funnel/docs/compute/pbs-torque.md | 11 +- docs/tools/funnel/docs/compute/slurm.md | 8 +- docs/tools/funnel/docs/databases/datastore.md | 2 +- docs/tools/funnel/docs/databases/postgres.md | 42 ---- .../funnel/docs/development/developers.md | 6 +- docs/tools/funnel/docs/metrics/prometheus.md | 1 - docs/tools/funnel/docs/security/advanced.md | 3 +- docs/tools/funnel/docs/security/basic.md | 9 +- docs/tools/funnel/docs/storage/http.md | 42 +--- .../funnel/docs/storage/mounted-inputs.md | 114 --------- docs/tools/funnel/index.md | 12 - zensical.toml | 27 +-- 16 files changed, 53 insertions(+), 507 deletions(-) delete mode 100644 docs/tools/funnel/docs/compute/kubernetes-storage.md delete mode 100644 docs/tools/funnel/docs/databases/postgres.md delete mode 100644 docs/tools/funnel/docs/storage/mounted-inputs.md diff --git a/docs/tools/funnel/.funnel-docs.json b/docs/tools/funnel/.funnel-docs.json index c6f72c86..7c6f0431 100644 --- a/docs/tools/funnel/.funnel-docs.json +++ b/docs/tools/funnel/.funnel-docs.json @@ -2,39 +2,7 @@ "source": "https://github.com/calypr/funnel", "branch": "develop", "revision": "a3be3b208e0f6202bc2a33d1946fb56a8e850431", - "files": [ - "compute/aws-batch.md", - "compute/deployment.md", - "compute/gcp-batch.md", - "compute/grid-engine.md", - "compute/htcondor.md", - "compute/kubernetes-storage.md", - "compute/kubernetes.md", - "compute/pbs-torque.md", - "compute/slurm.md", - "databases/boltdb.md", - "databases/datastore.md", - "databases/dynamodb.md", - "databases/elasticsearch.md", - "databases/mongodb.md", - "databases/postgres.md", - "development/developers.md", - "events/kafka.md", - "integrations/nextflow.md", - "integrations/py-tes.md", - "metrics/prometheus.md", - "security/advanced.md", - "security/basic.md", - "security/oauth2.md", - "storage/ftp.md", - "storage/google-storage.md", - "storage/http.md", - "storage/local.md", - "storage/mounted-inputs.md", - "storage/s3.md", - "storage/swift.md", - "tasks.md" - ], + "files": [], "pending_reconciliation": { "compute/gcp-batch.md": "a1ad1e72c4b7b8befd4ec98814855aa43fb48f8b76b6b12f9147ad64802f4d79", "compute/kubernetes.md": "456c6cbed9d9d1687b717dde7e647daea896fd96b03662ec4022d862930ca400", diff --git a/docs/tools/funnel/docs/compute/grid-engine.md b/docs/tools/funnel/docs/compute/grid-engine.md index 999e84e7..f885c55b 100644 --- a/docs/tools/funnel/docs/compute/grid-engine.md +++ b/docs/tools/funnel/docs/compute/grid-engine.md @@ -1,27 +1,29 @@ --- title: Grid Engine -menu: - main: - parent: Compute - weight: 20 --- # Grid Engine -Funnel can be configured to submit workers to [Grid Engine][ge] by making calls +Funnel can be configured to submit workers to [Grid Engine](https://gridscheduler.sourceforge.net/) by making calls to `qsub`. The Funnel server needs to run on a submission node. Configure Funnel to use Grid Engine by including the following config: It is recommended to update the submit file template so that the -`funnel worker run` command takes a config file as an argument -(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`) +`funnel worker run` command takes a config file as an argument: + +``` +{% raw %} +funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}} +{% endraw %} +``` ```YAML +{% raw %} Compute: gridengine GridEngine: - Template: | + Template: | #!/bin/bash #$ -N {{.TaskId}} #$ -o {{.WorkDir}}/funnel-stdout @@ -35,9 +37,10 @@ GridEngine: {{if ne .DiskGb 0.0 -}} {{printf "#$ -l h_fsize=%.0fG" .DiskGb}} {{- end}} - funnel worker run --taskID {{.TaskId}} +{% endraw %} ``` + The following variables are available for use in the template: | Variable | Description | @@ -51,4 +54,3 @@ The following variables are available for use in the template: See https://golang.org/pkg/text/template for information on creating templates. -[ge]: http://gridscheduler.sourceforge.net/documentation.html diff --git a/docs/tools/funnel/docs/compute/htcondor.md b/docs/tools/funnel/docs/compute/htcondor.md index 21c87873..a6c8ebdd 100644 --- a/docs/tools/funnel/docs/compute/htcondor.md +++ b/docs/tools/funnel/docs/compute/htcondor.md @@ -14,14 +14,15 @@ The Funnel server needs to run on a submission node. Configure Funnel to use HTCondor by including the following config: It is recommended to update the submit file template so that the -`funnel worker run` command takes a config file as an argument -(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`) +`funnel worker run` command takes a config file as an argument {% raw %} +(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`){% endraw %} ```YAML +{% raw %} Compute: htcondor HTCondor: - Template: | + Template: | universe = vanilla getenv = True executable = funnel @@ -42,6 +43,7 @@ HTCondor: {{- end}} queue +{% endraw %} ``` The following variables are available for use in the template: diff --git a/docs/tools/funnel/docs/compute/kubernetes-storage.md b/docs/tools/funnel/docs/compute/kubernetes-storage.md deleted file mode 100644 index e9c4fff9..00000000 --- a/docs/tools/funnel/docs/compute/kubernetes-storage.md +++ /dev/null @@ -1,219 +0,0 @@ ---- -title: Kubernetes storage -menu: - main: - parent: Compute - weight: 21 ---- - -# Kubernetes storage - -`Kubernetes.Storage` configures PVC-backed Kubernetes volumes for task workspaces and worker mounts. The `volumes` and `volumeMounts` fields use native Kubernetes objects, and configured volumes must use PVC sources. The `volumeClaimTemplates` and `persistentVolumeTemplates` fields contain native resource objects. `WorkspaceVolume` names the PVC-backed volume Funnel uses for the task workspace. `WorkerContainer` and `ExecutorContainer` select the containers that receive generated mounts when a Pod template has more than one regular container. - -When `Storage` is absent, Funnel keeps the existing `PVTemplate`, `PVCTemplate`, and Job template behavior. Explicit storage does not change TES inputs, outputs, or task paths. Configured GenericS3 mounted reads and writes require `Kubernetes.Storage` and a submitted storage plan. - -## Use an existing PVC - -Use `persistentVolumeClaim.claimName` for a claim that an operator already manages. Funnel does not create or delete that claim or its PV. - -```yaml -Worker: - WorkDir: /opt/funnel/funnel-work-dir -Kubernetes: - JobsNamespace: funnel-tasks - Storage: - WorkspaceVolume: workspace - volumes: - - name: workspace - persistentVolumeClaim: - claimName: shared-workspace -``` - -The claim must exist in `JobsNamespace` and support filesystem access from the worker and executor Pods. Funnel creates a task-specific workspace directory on that claim. `ReadWriteMany` supports placement across nodes. `ReadWriteOnce` requires compatible Pod placement. `ReadWriteOncePod` cannot serve the worker and executor Pods and is rejected. - -Use `volumes` entries for existing claims. Set `persistentVolumeClaim.claimName` to the claim's literal name. A PVC template creates a volume with the same logical name as the template. Do not add a matching `volumes` entry. A `volumeMounts` entry refers to either kind of volume through its `name` field. - -## Provision a PVC through a StorageClass - -Each PVC template creates a claim for one task and a volume with the same logical name. Set `WorkspaceVolume` to that name to use the claim for the task workspace. - -```yaml -Kubernetes: - JobsNamespace: funnel-tasks - Storage: - WorkspaceVolume: workspace - volumeClaimTemplates: - - metadata: - name: workspace - spec: - storageClassName: research-rwx - accessModes: - - ReadWriteMany - resources: - requests: - storage: "{{.DiskGb}}Gi" -``` - -For a task with ID `task-a`, this template creates the PVC `task-a-workspace` and a volume named `workspace`. Funnel sets the worker Job as the claim's owner. Kubernetes garbage collection removes the claim after the Job. The StorageClass determines provisioning and reclaim behavior. This configuration makes no PV API calls during normal submission or cleanup. - -Set each PVC template's `metadata.name` to a static logical volume name that is a DNS label, such as `workspace`. Funnel names the PVC `-`. Do not put `TaskId` in the PVC template's `metadata.name`. Omit `apiVersion` and `kind` to use `v1` and `PersistentVolumeClaim` defaults. Funnel infers an omitted `metadata.namespace` from the effective job namespace, which uses `JobsNamespace` or falls back to `Namespace`. Do not set `generateName`, owner references, or block volume mode. Each logical name must be unique and must not collide with a `volumes` entry. An existing-claim volume cannot refer to a generated claim name. - -Funnel renders Go templates in string values other than the PVC template's logical `metadata.name`, then decodes each object as a Kubernetes resource. It does not render an entire YAML document as template text. Quote templated quantity values so YAML keeps them as strings. Other fields can use `.TaskId`, `.Namespace`, and `.DiskGb`. They can also use `.Bucket`, `.Region`, and `.KmsKeyID` from the first GenericS3 entry when present. Those three values are empty when no GenericS3 entry exists. - -To migrate an existing task-specific PVC template, replace its `metadata.name` such as `work-{{.TaskId}}` with a static name such as `workspace`, then remove the `volumes` entry that points to the rendered claim name. If a PV template specifies `spec.claimRef.name`, remove that reference so Funnel fills the generated name, or change it to `{{.TaskId}}-workspace` for this example. Keep Go templates in other resource fields, including `spec.volumeName` and PV `metadata.name`. Existing version 4 storage plans remain compatible. The new naming rule applies to new task submissions. - -## Create an explicit PV and PVC pair - -Set `PVC.spec.volumeName` to the exact `metadata.name` of a `persistentVolumeTemplates` entry. The PVC template keeps its static logical name, while `spec.volumeName` can use `TaskId` to select a task-specific PV. Funnel fills a missing PV `spec.claimRef` with the generated PVC name and namespace. For task `task-a`, the example claim is `task-a-workspace`, and the PV claim reference uses that name. If the template provides a conflicting reference, Funnel rejects it before creating resources. - -```yaml -Kubernetes: - JobsNamespace: funnel-tasks - Storage: - WorkspaceVolume: workspace - volumeClaimTemplates: - - metadata: - name: workspace - spec: - storageClassName: "" - volumeName: disk-{{.TaskId}} - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi - persistentVolumeTemplates: - - apiVersion: v1 - kind: PersistentVolume - metadata: - name: disk-{{.TaskId}} - spec: - capacity: - storage: 20Gi - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Retain - storageClassName: "" - hostPath: - path: /srv/funnel/{{.TaskId}} -``` - -The `hostPath` source makes this example suitable only for a local cluster whose worker and executor Pods use the same node and path. Use the deployment's native CSI or other PV source in a real cluster. Each PV template must be a cluster-scoped `v1` `PersistentVolume` with an explicit `metadata.name` that includes `TaskId`. Do not set `metadata.namespace`, `generateName`, owner references, or block volume mode. A PV template must have exactly one native PV source and link to exactly one PVC template. - -Funnel creates the PV and PVC with task ownership and preserves the existing cleanup lifecycle. Explicit PV storage adds a finalizer to the worker Job. Cleanup requests deletion of owned PVs and the Job before releasing that finalizer. A failed delete request leaves the plan available for retry. Reconciliation retries cleanup for canceled tasks even when the Job is still running or its deletion request failed. Submission checks persisted cancellation before and after resource creation and cleans up resources created during cancellation. The server needs permission to delete these PVs. Finish PV-backed tasks and clean up their resources before removing cluster-scoped deletion permissions. Kubernetes storage-protection finalizers and the storage driver's normal cleanup still apply. - -Before creating the worker Job, Funnel checks PV permissions with Kubernetes `SelfSubjectAccessReview`. Explicit PV templates require cluster-scoped `create` permission and `get` and `delete` permission for each generated PV name. The check uses the effective task configuration, including plugin overrides. A denied or unavailable authorization check rejects submission before creating the Job or its storage. Existing-claim and StorageClass-only configurations skip these PV checks. - -The Funnel server identity must be allowed to create `selfsubjectaccessreviews` in the `authorization.k8s.io` API group. Kubernetes normally grants this through its default authenticated-user roles. The submission check does not require PV `list` permission; the separate orphan cleanup sweep still needs it. Authorization can change after the check, so keep PV permissions until all PV-backed tasks have been cleaned up. - -To enable explicit PV storage, an administrator can apply the following RBAC resources. Replace the subject name and namespace with the service account used by the Funnel server Pod. These permissions supplement the server's existing namespaced permissions for Jobs, Pods, and PVCs. - -```yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: funnel-pv-manager -rules: - - apiGroups: [""] - resources: [persistentvolumes] - verbs: [create, get, delete] - - apiGroups: [authorization.k8s.io] - resources: [selfsubjectaccessreviews] - verbs: [create] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: funnel-pv-manager -subjects: - - kind: ServiceAccount - name: funnel - namespace: funnel-tasks -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: funnel-pv-manager -``` - -## Add worker mounts - -`volumeMounts` contains native Kubernetes `volumeMount` objects. Use the configured volume's `name`. You can mount the same volume at multiple paths. - -```yaml -Kubernetes: - JobsNamespace: funnel-tasks - Storage: - WorkspaceVolume: workspace - volumes: - - name: workspace - persistentVolumeClaim: - claimName: shared-workspace - - name: reference-data - persistentVolumeClaim: - claimName: reference-datasets - volumeMounts: - - name: reference-data - mountPath: /mnt/reference - readOnly: true - subPath: genomes - - name: reference-data - mountPath: /mnt/annotations - readOnly: true - subPath: annotations -``` - -Funnel adds `volumeMounts` to the selected worker container. It generates the workspace mount for the worker and executor. Extra worker mounts do not expose data automatically at TES input paths. A storage backend must still prepare those inputs. Worker mount paths cannot overlap the task workspace, each other, or mounts already in the Job template. - -GenericS3 `MountedRead` and `MountedWrite` select a worker mount by volume name and, optionally, its static `subPath`. If one `volumeMounts` entry uses the selected volume, omit `SubPath`. If several entries use that volume, set `SubPath` to the exact `volumeMounts[].subPath` value. Set `SubPath: ""` to select a static root mount. A mount that uses `subPathExpr` can be selected only when it is the unique mount for that volume and `SubPath` is absent. - -The selected mount's authored `mountPath` is the S3 root. Funnel does not append the selected `SubPath` or the S3 `Prefix` to that path. A write mount must be writable. The underlying claim must also permit writes. - -## Configure Job templates - -With explicit storage, Funnel renders Job templates with `NeedsPVC: false` and adds resolved volumes and mounts afterward. Templates that already condition legacy storage on `NeedsPVC` keep their other settings. Remove template-defined volumes or mount paths that conflict with configured storage. - -Funnel selects the sole regular container by default. For templates with sidecars, set `Storage.WorkerContainer` and `Storage.ExecutorContainer` to the main container names. Funnel adds storage and worker configuration only to those containers. Init containers and sidecars keep their template-defined mounts. - -The worker Job name must equal `TaskId`. `Worker.WorkDir` must be absolute. Per-task plugin overrides cannot change `JobsNamespace`, because a backend reconciles one configured Jobs namespace. - -`Kubernetes.WorkerImage` selects the image supplied to the worker Job template and skips server Pod discovery. When it is absent, Funnel keeps the existing image discovery behavior. Templates use `{{.Image}}` to consume that setting. - -## Ownership and upgrades - -Funnel labels resources it creates and records the worker Job UID. Creation retries accept only resources with matching ownership and specifications. Funnel rejects an existing resource with a conflicting owner or specification instead of adopting it. Operator-owned claims remain under operator control. - -Provisioning checks the worker Job before and after each resource creation. If cancellation removes the Job during creation, Funnel deletes a late-created owned PV. A process crash, an ambiguous create response, or a failed compensating deletion can leave an orphan. Recovery requires an orphan cleanup sweep with PV scanning enabled by a PV template. An existing-claim-only configuration does not scan cluster-scoped PVs. - -Funnel records a version 4 plan in the Job's `funnel.calypr.org/storage-plan` annotation and supplies the same plan through `FUNNEL_KUBERNETES_STORAGE_PLAN`. The plan records resolved volumes, worker mounts, S3 routes, claim names, namespace, and work directory. It contains no credentials. Existing Jobs with version 4 plans remain compatible. Do not set the plan environment variable or annotation in templates. - -If the annotation is missing, cleanup recovers the plan from the Job's worker environment. It validates the task and namespace before using that copy. Conflicting copies, invalid plans, or a storage-finalized Job with neither copy cause an error and retain the finalizer. Restore the correct plan before retrying cleanup in those cases. - -Cleanup validates resource ownership separately from S3 transfer routes. Version 4 plans with write-route overlaps rejected by newer workers can still be cleaned up. New submissions and workers reject those overlaps. - -The broader native storage migration changes the config schema. Migrate configurations to native objects and rebuild Go RPC plugins against the updated config package before exchanging configuration with them. This PVC naming change adds no config fields and requires no plugin rebuild by itself. The schema preserves `Kubernetes.ForbiddenPathPrefixes` at field 20 and uses `WorkerImage` at 24 and `Storage` at 25. It reserves `KubernetesStorage` field 2 and the names `Bindings` and `MountedInputs`. `MountedRead` and `MountedWrite` reserve field 1 and the name `WorkerMount`. Kubernetes field numbers 21–23 and the names `PVCMode`, `StorageClassName`, `SharedPVCName`, and `S3FilesFilesystemId` are also reserved. Configuration with retired names fails parsing. - -For configurations from PR #1454, replace `PVCMode: full` with native PV and PVC templates, `PVCMode: pvc` with a PVC template containing `spec.storageClassName`, and `PVCMode: shared` with `persistentVolumeClaim.claimName`. Keep `WorkerImage` as an independent field. - -Chicago's `S3FilesFilesystemId` used field 20, which conflicts with upstream `ForbiddenPathPrefixes`. Do not decode old Chicago binary configuration with the upstream schema. Both fields use a length-delimited wire representation, so a decoder cannot distinguish them. Export the configuration with its original schema, move the filesystem ID into the deployment's native CSI PV template, and load the migrated named configuration. Rebuild plugins against the canonical schema before exchanging configuration. Funnel supplies no automatic decoder for the conflicting binary format. - -The version 4 decoder cannot run or clean up Jobs with version 1, 2, or 3 plans. Finish those tasks and clean up their Jobs with a compatible older binary before upgrading. Jobs without a storage plan still require the legacy cleanup configuration during migration. These requirements apply whether the task uses existing claims or native PVC templates. - -Helm resolves deployment values before Funnel renders task variables. Charts must preserve literal Funnel expressions such as `{{.TaskId}}` through Helm rendering, as they do for existing Funnel Job templates. - -## Verify against a local cluster - -The opt-in integration test uses an existing Kubernetes context, creates temporary namespaces, and runs worker and executor Jobs with `alpine:3.21`. Use a disposable single-node cluster with a `standard` StorageClass and access to that image. The test checks file exchange through a provisioned claim, an existing claim, and an explicit hostPath PV and PVC pair. It then verifies ownership-based cleanup, including PV protection and Job finalizers. - -```sh -FUNNEL_KUBERNETES_TEST_KUBECONFIG=/path/to/test-kubeconfig \ - go test ./compute/kubernetes -run '^TestNativeStorageLive$' -v -count=1 -``` - -The test verifies real Kubernetes mounts and cleanup. It does not certify an S3 CSI driver or asynchronous S3 output publication. - -To verify permission failures, run the RBAC test with an administrator kubeconfig for a disposable cluster. It creates temporary service accounts and roles, impersonates those accounts, and checks that denied PV permissions leave no worker Job or storage resources. It also checks successful submission and cleanup with PV permissions restricted by name and with a PVC-only configuration. - -```sh -FUNNEL_KUBERNETES_TEST_KUBECONFIG=/path/to/test-kubeconfig \ - go test ./compute/kubernetes -run '^TestNativeStoragePermissionsLive$' -v -count=1 -``` diff --git a/docs/tools/funnel/docs/compute/pbs-torque.md b/docs/tools/funnel/docs/compute/pbs-torque.md index 5a7751f5..59b2dfa1 100644 --- a/docs/tools/funnel/docs/compute/pbs-torque.md +++ b/docs/tools/funnel/docs/compute/pbs-torque.md @@ -1,5 +1,6 @@ --- title: PBS/Torque +render_macros: true menu: main: parent: Compute @@ -14,14 +15,15 @@ The Funnel server needs to run on a submission node. Configure Funnel to use PBS by including the following config: It is recommended to update the submit file template so that the -`funnel worker run` command takes a config file as an argument -(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`) +`funnel worker run` command takes a config file as an argument +(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {% raw %}{{.TaskId}}{% endraw %}`) +{% raw %} ```YAML Compute: pbs PBS: - Template: | + Template: | #!/bin/bash #PBS -N {{.TaskId}} #PBS -o {{.WorkDir}}/funnel-stdout @@ -38,6 +40,7 @@ PBS: funnel worker run --taskID {{.TaskId}} ``` +{% endraw %} The following variables are available for use in the template: | Variable | Description | @@ -51,4 +54,4 @@ The following variables are available for use in the template: See https://golang.org/pkg/text/template for information on creating templates. -[pbs]: http://www.adaptivecomputing.com/products/open-source/torque/ +[pbs]: https://hpc-wiki.info/hpc/Torque diff --git a/docs/tools/funnel/docs/compute/slurm.md b/docs/tools/funnel/docs/compute/slurm.md index 6bb1243f..98f16977 100644 --- a/docs/tools/funnel/docs/compute/slurm.md +++ b/docs/tools/funnel/docs/compute/slurm.md @@ -14,14 +14,15 @@ The Funnel server needs to run on a submission node. Configure Funnel to use Slurm by including the following config: It is recommended to update the submit file template so that the -`funnel worker run` command takes a config file as an argument -(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`) +`funnel worker run` command takes a config file as an argument +(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {% raw %}{{.TaskId}}{% endraw %}`) +{% raw %} ```YAML Compute: slurm Slurm: - Template: | + Template: | #!/bin/bash #SBATCH --job-name {{.TaskId}} #SBATCH --ntasks 1 @@ -39,6 +40,7 @@ Slurm: funnel worker run --taskID {{.TaskId}} ``` +{% endraw %} The following variables are available for use in the template: | Variable | Description | diff --git a/docs/tools/funnel/docs/databases/datastore.md b/docs/tools/funnel/docs/databases/datastore.md index 9f39d1cd..ea31d8c6 100644 --- a/docs/tools/funnel/docs/databases/datastore.md +++ b/docs/tools/funnel/docs/databases/datastore.md @@ -38,7 +38,7 @@ to support the task-list queries. This is typically done through command-line by referencing an **index.yaml** file (do not change the filename) with the following content: -```sh +```shell gcloud datastore indexes create path/to/index.yaml --database='funnel' ``` diff --git a/docs/tools/funnel/docs/databases/postgres.md b/docs/tools/funnel/docs/databases/postgres.md deleted file mode 100644 index b91c7952..00000000 --- a/docs/tools/funnel/docs/databases/postgres.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: Postgres -menu: - main: - parent: Databases ---- - -# Postgres - -> [!WARNING] -> -> Postgres support is in early development. -> -> Production deployments are recommended to use the alternative storage backends available, such as MongoDB, ElasticSearch, DynamoDB, etc. - -## Configuration - -```yaml -Database: postgres - -Postgres: - Host: localhost - Database: funnel - User: funnel - Password: example -``` - -## Default Values - -```go - Postgres: &Postgres{ - Host: "localhost:5432", - Database: "funnel", - User: "funnel", - Password: "example", - Timeout: &TimeoutConfig{ - TimeoutOption: &TimeoutConfig_Duration{ - Duration: durationpb.New(time.Second * 30), - }, - }, - }, -``` diff --git a/docs/tools/funnel/docs/development/developers.md b/docs/tools/funnel/docs/development/developers.md index 7769d901..c18a4ec1 100644 --- a/docs/tools/funnel/docs/development/developers.md +++ b/docs/tools/funnel/docs/development/developers.md @@ -17,7 +17,7 @@ This page contains a rough collection of notes for people wanting to build Funne 2. Ensure GOPATH is set. See [the docs][gopath] for help. Also, you probably want to add `$GOPATH/bin` to your `PATH`. 3. Clone funnel and build - ```sh + ```shell $ git clone https://github.com/ohsu-comp-bio/funnel.git $ cd funnel $ make @@ -36,7 +36,7 @@ A Funnel development environment includes: - [gRPC Gateway][gateway] for HTTP communication. - [Angular][angular] and [SASS][sass] for the web dashboard. - [GNU Make][make] for development tasks. -- [Docker][docker] for executing task containers (tested with v1.12, v1.13). +- [Docker](https://docker.io) for executing task containers (tested with v1.12, v1.13). - [dep][dep] for Go dependency vendoring. - [Make][make] for development/build commands. - [NodeJS][node] and [NPM][npm] for web dashboard development. @@ -64,8 +64,6 @@ mock interfaces in test code, for example, to mock the Google Cloud APIs. [grpc]: http://www.grpc.io/ [sass]: http://sass-lang.com/ [make]: https://www.gnu.org/software/make/ -[docker]: https://docker.io -[python]: https://www.python.org/ [dep]: https://golang.github.io/dep/ [node]: https://nodejs.org [npm]: https://www.npmjs.com/ diff --git a/docs/tools/funnel/docs/metrics/prometheus.md b/docs/tools/funnel/docs/metrics/prometheus.md index 1b3495b6..c59894b4 100644 --- a/docs/tools/funnel/docs/metrics/prometheus.md +++ b/docs/tools/funnel/docs/metrics/prometheus.md @@ -32,5 +32,4 @@ Funnel exports these metrics: of bytes of disk space available by all nodes. [prom]: https://prometheus.io/ -[gauge]: https://prometheus.io/docs/concepts/metric_types/#gauge [graf]: https://grafana.com/ diff --git a/docs/tools/funnel/docs/security/advanced.md b/docs/tools/funnel/docs/security/advanced.md index 3864e344..8812d759 100644 --- a/docs/tools/funnel/docs/security/advanced.md +++ b/docs/tools/funnel/docs/security/advanced.md @@ -21,8 +21,7 @@ In this way this Wrapper can manage the bucket and the keys (the Wrapper would Stay tuned for this feature's development! This feature is being tracked with the following: -- GitHub Branch: https://github.com/ohsu-comp-bio/funnel/tree/feature/credentials -- Pull Request: https://github.com/ohsu-comp-bio/funnel/pull/1098 +- Pull Request: https://github.com/calypr/funnel/pull/1098 # Credits 🙌 diff --git a/docs/tools/funnel/docs/security/basic.md b/docs/tools/funnel/docs/security/basic.md index 7a2d9080..0b19e071 100644 --- a/docs/tools/funnel/docs/security/basic.md +++ b/docs/tools/funnel/docs/security/basic.md @@ -40,15 +40,10 @@ so you will also need to configure the RPC client. ```yaml RPCClient: - Credential: - User: funnel - Password: abc123 + User: funnel + Password: abc123 ``` -Beginning with Funnel v0.11.8, `User` and `Password` must be nested under -`RPCClient.Credential`. Configurations created for earlier releases need to be -updated to use this structure. - Make sure to properly protect the configuration file so that it's not readable by everyone: diff --git a/docs/tools/funnel/docs/storage/http.md b/docs/tools/funnel/docs/storage/http.md index 740d12b9..81922053 100644 --- a/docs/tools/funnel/docs/storage/http.md +++ b/docs/tools/funnel/docs/storage/http.md @@ -7,36 +7,9 @@ menu: # HTTP(S) -Funnel can download inputs from and upload outputs to HTTP(S) URLs that need no -credentials: public URLs, or presigned URLs issued by a cloud storage provider -such as Amazon S3 or Google Cloud Storage. No authentication mechanism is -allowed, so the URL itself must grant access. - -- Inputs are downloaded with a GET request. -- Outputs are uploaded with a PUT request that carries a `Content-Length` - header, as presigned S3 upload URLs require. - -Presigned URLs work for `FILE` inputs and outputs. A presigned URL is signed for -one exact object, but Funnel derives the URL of each file in a `DIRECTORY` -output, and of each match of a wildcard output, by appending the file's path to -the path of the output URL, keeping its query string. Those derived URLs carry -no valid signature, so use one `FILE` output per presigned URL. - -For `DIRECTORY` and wildcard outputs, point the output URL at a server that -signs each upload as it arrives, when the file name is known, and redirects it -to the storage provider: - -- An upload follows a 307 or 308 redirect and resends the file to the new URL. -- Uploads send `Expect: 100-continue` and wait up to one second for the server - to answer before sending the file. A server that redirects within that second - doesn't receive the file; a slower one may receive some or all of it first. -- A 301, 302 or 303 response to an upload is reported as an error, because - following it would turn the upload into a GET. -- Downloads follow redirects too. - -The URL is stored with the task, including any signature in its query string, -so issue presigned URLs with a short expiry. The storage provider limits the -size of a single PUT (5 GB on Amazon S3). +Funnel supports downloading files from public URLs via GET requests. No authentication +mechanism is allowed. This backend can be used to fetch objects from cloud storage +providers exposed using presigned URLs. The HTTP storage client is enabled by default, but may be explicitly disabled in the worker config: @@ -44,7 +17,7 @@ worker config: ```yaml HTTPStorage: Disabled: false - # Timeout for http(s) requests, including the transfer itself. + # Timeout for http(s) GET requests. Timeout: 30s ``` @@ -56,14 +29,9 @@ HTTPStorage: "url": "http://fakedomain.com/hello.txt", "path": "/inputs/hello.txt" }], - "outputs": [{ - "url": "https://my-bucket.s3.amazonaws.com/hello-upper.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=...", - "path": "/outputs/hello-upper.txt", - "type": "FILE" - }], "executors": [{ "image": "alpine", - "command": ["sh", "-c", "tr a-z A-Z < /inputs/hello.txt > /outputs/hello-upper.txt"] + "command": ["cat", "/inputs/hello.txt"], }] } ``` diff --git a/docs/tools/funnel/docs/storage/mounted-inputs.md b/docs/tools/funnel/docs/storage/mounted-inputs.md deleted file mode 100644 index e8b665cb..00000000 --- a/docs/tools/funnel/docs/storage/mounted-inputs.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: Mounted inputs -menu: - main: - parent: Storage ---- - -# Mounted inputs - -`GenericS3.MountedRead` and `GenericS3.MountedWrite` let a Kubernetes worker read inputs or publish outputs through PVC-backed filesystem mounts. Each setting names a configured storage volume. If that volume has multiple worker mounts, use `SubPath` to select the mount whose static Kubernetes `subPath` matches exactly. Read and write routes are optional and can use separate volumes or mount views. - -```yaml -GenericS3: - - Disabled: false - Endpoint: https://s3.example.org - Bucket: analysis-data - Key: storage-key - Secret: storage-secret - Region: us-east-1 - MountedRead: - Volume: reference-data - SubPath: genomes - Prefix: genomes/ - MountedWrite: - Volume: result-data - Prefix: results/ - Timeout: 2700s - -Kubernetes: - JobsNamespace: funnel-tasks - Storage: - WorkspaceVolume: workspace - volumes: - - name: workspace - persistentVolumeClaim: - claimName: task-workspace - - name: reference-data - persistentVolumeClaim: - claimName: reference-pvc - - name: result-data - persistentVolumeClaim: - claimName: task-results - volumeMounts: - - name: reference-data - mountPath: /mnt/reference - readOnly: true - subPath: genomes - - name: reference-data - mountPath: /mnt/annotations - readOnly: true - subPath: annotations - - name: result-data - mountPath: /mnt/results -``` - -The two reference mounts use one PVC at different paths. `MountedRead.SubPath: genomes` selects the first view. `MountedWrite` omits `SubPath` because `result-data` has one worker mount. If a volume has several mounts, set `SubPath` to the exact static `volumeMounts[].subPath` value. Set `SubPath: ""` to select a static root mount. A mount with `subPathExpr` can be selected only when it is the unique mount for that volume and `SubPath` is absent. - -`Prefix` is relative to the bucket. An empty prefix selects the whole bucket. The selected mount's authored `mountPath` is the route root. For example, `s3://analysis-data/genomes/genome.fa` maps to `/mnt/reference/genome.fa`. Funnel does not append the selected subpath or prefix to `mountPath`. The storage driver and operator must make the mounted view correspond to the configured bucket prefix and provide the required freshness. - -`MountedWrite` uses the same volume, subpath, and prefix rules. Its selected mount must be writable, and the underlying PVC must permit writes. Before creating task resources, Funnel checks both existing claims and PVC templates for `ReadWriteOnce`, `ReadWriteMany`, or `ReadWriteOncePod` access. A claim with only `ReadOnlyMany` cannot serve mounted outputs. The workspace still cannot use `ReadWriteOncePod` because worker and executor Pods share it. Funnel appends an output key path relative to the configured prefix when it copies a TES output file into the mount. - -For native storage configuration details, including existing claims, StorageClass-provisioned claims, and explicit PV/PVC pairs, see [Kubernetes storage](../../compute/kubernetes-storage/). - -## TES behavior - -Clients continue to submit ordinary TES inputs. They do not supply PVC names, volume names, worker mount paths, or deployment paths. - -```json -{ - "inputs": [ - { - "url": "s3://analysis-data/genomes/genome.fa", - "path": "/inputs/genome.fa", - "type": "FILE" - } - ], - "executors": [ - { - "image": "alpine:3.21", - "command": ["cat", "/inputs/genome.fa"] - } - ] -} -``` - -The worker copies each input into the task workspace. For a matching GenericS3 mounted read, it reads the source from the selected filesystem mount. The executor sees the copy at `/inputs/genome.fa`. Changing that copy does not change the mounted source. Directory inputs preserve their relative filenames. Reference mounts remain worker-only. They do not change TES `volumes`, which retain their empty scratch-directory behavior. - -## Read routing and safety - -The worker activates a mounted read or write only when the submitted storage plan names the exact GenericS3 `Endpoint` and `Bucket`. The plan stores `S3ReadMounts` and `S3WriteMounts` with the endpoint, bucket, prefix, and mount root, without credentials. The worker ignores static `MountedRead` and `MountedWrite` settings, including when the submitted plan has no mounted routes. Configured mounted reads and writes require `Kubernetes.Storage` and its submitted plan. The API server's `NewMux` does not activate filesystem reads or writes. - -GenericS3 handles inputs whose bucket-relative key matches its configured prefix. Funnel treats S3 keys literally. Spaces, `#`, and percent signs remain part of the key. For example, `a%20b.txt` names the literal file `a%20b.txt`. Prefix matching uses whole path components, so `genomes/` does not match `genomes-old/file.fa`. Unmatched URLs use the configured storage API path. - -A matched file must exist beneath the selected mount root. Funnel rejects traversal, encoded traversal forms, and escaping symlinks through rooted filesystem operations. Regular-file symlinks within the root can be read. Directory listings reject directory symlinks and special files. A missing or unsafe matched input fails the transfer. Funnel does not fall back to the API for that input. - -## Configuration ownership - -The server resolves each backend's selected worker volume mount and records mounted reads and writes in version 4 storage plans. Workers use the plan directly, even when their static ConfigMap differs. The server does not read worker filesystem paths. Existing Jobs with version 4 plans remain compatible. The version 4 decoder cannot run or clean up Jobs with version 1, 2, or 3 plans. Finish those tasks and clean up their Jobs with a compatible older binary before upgrading. - -The broader native storage migration changes the config schema. Replace `MountedRead.WorkerMount` and `MountedWrite.WorkerMount` with `Volume`, and set `SubPath` when one volume has multiple explicit worker mounts. Rebuild Go RPC plugins against the updated config package before exchanging configuration with them. The automatic PVC naming change does not alter the plugin schema or require another rebuild. The schema reserves field 1 and the name `WorkerMount` in `MountedRead` and `MountedWrite`. Other compute backends retain their existing storage clients. No TES schema fields change. - -## Output publication - -For an output URL that matches a configured `MountedWrite` endpoint, bucket, and prefix, the worker copies each output file to a private temporary file beside its destination. After closing the file, it renames the temporary file over the destination. Concurrent transfers publish complete files instead of copying into the same inode. If the source already is the mounted destination, Funnel skips the copy. It then calls S3 `HeadObject` until the object exists with the same size as the local file. The worker checks zero-byte files too. - -Mounted file transfers require Unix. The filesystem must support exclusive temporary-file creation, POSIX file permissions, and atomic replacement through same-directory rename, including across worker Pods. A failed rename fails the transfer. Funnel does not retry through an in-place copy or an API upload. Separate write routes must use nonoverlapping filesystem views. Funnel rejects overlapping roots and overlapping views of the same PVC that it can identify from the storage plan. - -After a failed or canceled copy, Funnel attempts to remove its temporary file and reports cleanup errors. A process crash can leave temporary files, and a storage driver can export temporary names to the bucket before Funnel removes them. Account for these files in the deployment's storage cleanup policy. Filesystem replacement does not make the driver's later S3 synchronization atomic. See [S3 Files synchronization](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-files-synchronization.html) for that driver's rename and export behavior. - -This check proves that S3 reports an object with the expected size. It does not compare file contents, checksums, or object versions. A same-sized object left by an earlier task can satisfy the check. Set `MountedWrite.Timeout` to bound polling. The example's `2700s` timeout is 45 minutes. The default is three hours, and the worker polls every two seconds. An explicit timeout must be positive. Task cancellation also stops the wait. - -The GenericS3 credentials must allow `HeadObject` for mounted destinations. They do not need upload permission for an output handled by the mount. An output URL that does not match the configured endpoint, bucket, and prefix follows the regular GenericS3 API upload path, which requires upload permission. Outputs do not use a `MountedRead` setting. - -Funnel handles directory outputs as individual files. It records successful files in TES output logs and preserves partial successes when another file fails. A timeout, cancellation, or transfer error prevents successful task completion. Retries retain one successful log entry per file after publication succeeds. diff --git a/docs/tools/funnel/index.md b/docs/tools/funnel/index.md index 800aeca1..05176f5e 100644 --- a/docs/tools/funnel/index.md +++ b/docs/tools/funnel/index.md @@ -8,18 +8,6 @@ Funnel provides task execution and workflow orchestration around the GA4GH TES m It is designed to streamline distributed batch processing by exposing a simple task API and a set of components that can be adapted to different compute and storage backends. -## Connect to a remote server - -Use `-S` to point a command at a Funnel server, or set `FUNNEL_SERVER` for your shell: - -```sh -funnel -S https://funnel.example.com task list -export FUNNEL_SERVER=https://funnel.example.com -funnel task list -``` - -The `-S` flag takes precedence over `FUNNEL_SERVER` when both are set. - ## Read Next - [Architecture Overview](docs/docs.md) diff --git a/zensical.toml b/zensical.toml index 004b97fa..8c148d27 100644 --- a/zensical.toml +++ b/zensical.toml @@ -101,37 +101,34 @@ nav = [ { "Tasks" = "tools/funnel/docs/tasks.md" }, { "Storage" = [ "tools/funnel/docs/storage/index.md", - { "FTP" = "tools/funnel/docs/storage/ftp.md" }, - { "Google Storage" = "tools/funnel/docs/storage/google-storage.md" }, - { "HTTP(S)" = "tools/funnel/docs/storage/http.md" }, { "Local" = "tools/funnel/docs/storage/local.md" }, - { "Mounted inputs" = "tools/funnel/docs/storage/mounted-inputs.md" }, - { "OpenStack Swift" = "tools/funnel/docs/storage/swift.md" }, { "S3" = "tools/funnel/docs/storage/s3.md" }, + { "Google Storage" = "tools/funnel/docs/storage/google-storage.md" }, + { "OpenStack Swift" = "tools/funnel/docs/storage/swift.md" }, + { "HTTP(S)" = "tools/funnel/docs/storage/http.md" }, + { "FTP" = "tools/funnel/docs/storage/ftp.md" }, ] }, { "Compute" = [ + { "Deploying a Cluster" = "tools/funnel/docs/compute/deployment.md" }, { "AWS Batch" = "tools/funnel/docs/compute/aws-batch.md" }, - { "Deploying a cluster" = "tools/funnel/docs/compute/deployment.md" }, { "GCP Batch" = "tools/funnel/docs/compute/gcp-batch.md" }, - { "Grid Engine" = "tools/funnel/docs/compute/grid-engine.md" }, - { "HTCondor" = "tools/funnel/docs/compute/htcondor.md" }, { "Kubernetes" = "tools/funnel/docs/compute/kubernetes.md" }, - { "Kubernetes storage" = "tools/funnel/docs/compute/kubernetes-storage.md" }, - { "PBS/Torque" = "tools/funnel/docs/compute/pbs-torque.md" }, + { "HTCondor" = "tools/funnel/docs/compute/htcondor.md" }, { "Slurm" = "tools/funnel/docs/compute/slurm.md" }, + { "PBS/Torque" = "tools/funnel/docs/compute/pbs-torque.md" }, + { "Grid Engine" = "tools/funnel/docs/compute/grid-engine.md" }, ] }, { "Databases" = [ - { "Datastore" = "tools/funnel/docs/databases/datastore.md" }, - { "DynamoDB" = "tools/funnel/docs/databases/dynamodb.md" }, - { "Elasticsearch" = "tools/funnel/docs/databases/elasticsearch.md" }, { "Embedded" = "tools/funnel/docs/databases/boltdb.md" }, { "MongoDB" = "tools/funnel/docs/databases/mongodb.md" }, - { "Postgres" = "tools/funnel/docs/databases/postgres.md" }, + { "Elasticsearch" = "tools/funnel/docs/databases/elasticsearch.md" }, + { "DynamoDB" = "tools/funnel/docs/databases/dynamodb.md" }, + { "Datastore" = "tools/funnel/docs/databases/datastore.md" }, ] }, { "Security" = [ - { "Advanced Auth" = "tools/funnel/docs/security/advanced.md" }, { "Basic Auth" = "tools/funnel/docs/security/basic.md" }, { "OAuth2" = "tools/funnel/docs/security/oauth2.md" }, + { "Advanced Auth" = "tools/funnel/docs/security/advanced.md" }, ] }, { "Integrations" = [ { "Nextflow" = "tools/funnel/docs/integrations/nextflow.md" },