diff --git a/.github/workflows/funnel-docs-check.yml b/.github/workflows/funnel-docs-check.yml new file mode 100644 index 0000000..c828f5a --- /dev/null +++ b/.github/workflows/funnel-docs-check.yml @@ -0,0 +1,29 @@ +name: Check Funnel documentation sync + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + name: Test sync and build site + runs-on: ubuntu-latest + steps: + - name: Check out CALYPR docs + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + - name: Test sync behavior + run: python -m unittest discover -s tests -v + - name: Install website dependencies + run: python -m pip install -r requirements.txt + - name: Build website + run: zensical build --clean diff --git a/.github/workflows/sync-funnel-docs.yml b/.github/workflows/sync-funnel-docs.yml new file mode 100644 index 0000000..147aaa7 --- /dev/null +++ b/.github/workflows/sync-funnel-docs.yml @@ -0,0 +1,77 @@ +name: Update Funnel documentation + +on: + schedule: + - cron: '17 6 * * *' + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: sync-funnel-docs + cancel-in-progress: false + +jobs: + sync: + name: Prepare Funnel docs update + runs-on: ubuntu-latest + steps: + - name: Check out CALYPR docs + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check out Funnel develop + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: calypr/funnel + ref: develop + path: .funnel-upstream + persist-credentials: false + + - name: Check Funnel revision + id: revision + run: | + revision=$(git -C .funnel-upstream rev-parse HEAD) + previous=$(python3 -c 'import json; print(json.load(open("docs/tools/funnel/.funnel-docs.json"))["revision"])') + echo "revision=$revision" >> "$GITHUB_OUTPUT" + if [ "$revision" != "$previous" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Python + if: steps.revision.outputs.changed == 'true' + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Test sync behavior + if: steps.revision.outputs.changed == 'true' + run: python -m unittest discover -s tests -v + + - name: Sync Funnel documentation + if: steps.revision.outputs.changed == 'true' + run: python scripts/sync_funnel_docs.py .funnel-upstream --revision "${{ steps.revision.outputs.revision }}" + + - name: Install website dependencies + if: steps.revision.outputs.changed == 'true' + run: python -m pip install -r requirements.txt + + - name: Build the website + if: steps.revision.outputs.changed == 'true' + run: zensical build --clean + + - name: Create or update pull request + if: steps.revision.outputs.changed == 'true' + uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7 + with: + branch: automation/sync-funnel-docs + commit-message: 'docs: sync Funnel develop documentation' + title: 'docs: sync Funnel develop documentation' + body: | + Update the CALYPR site from [Funnel develop](https://github.com/calypr/funnel/tree/${{ steps.revision.outputs.revision }}) at `${{ steps.revision.outputs.revision }}`. + + Review the generated content and navigation before merging. This PR was prepared and built by the scheduled documentation sync. + delete-branch: true diff --git a/.gitignore b/.gitignore index f74ee97..d9bc0c6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ venv site/ .idea/ attic/ + +/.funnel-upstream/ diff --git a/DEVELOPING.md b/DEVELOPING.md index 5f9a361..475eadd 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -1,176 +1,57 @@ # Developing CALYPR Docs -This repo now uses a Zensical-only build pipeline. MkDocs, `.nav.yml`, and the old multirepo plugin are gone. +The site is built with Zensical from the content in `docs/`. Navigation is +configured in `zensical.toml`. -The main thing to understand is that the published site is built from a generated docs tree, not directly from `docs/`. +## Build locally -## Mental Model - -- `docs/` contains repo-authored source content. -- `.generated/docs/` is the staged build input that Zensical actually reads. -- `site/` is the rendered output from `zensical build`. -- `zensical.toml` is the only navigation source of truth. - -The build pipeline is: - -1. Copy repo-authored docs from `docs/` into `.generated/docs/` -2. Import selected upstream docs from `git-drs` and `syfon` -3. Apply a few link rewrites needed for imported pages -4. Add generated assets like `termynal.css` and `termynal.js` -5. Run Zensical against `.generated/docs` - -## Core Commands - -From the repo root: - -```bash +```sh python3 -m venv venv source venv/bin/activate pip install -r requirements.txt -make serve +make build ``` -Useful commands: - -- `make prepare`: rebuild `.generated/docs` -- `make build`: run a clean Zensical build -- `make serve`: prepare docs and start the local server -- `make update-branches`: update GitHub source links in authored docs to the preferred upstream branches - -If you want the raw commands: - -```bash -python scripts/prepare_docs.py -zensical build --clean -zensical serve +Use `make serve` for local development. The rendered site is written to `site/`. +The [GitHub Pages workflow](.github/workflows/publish.yml) builds and publishes +when changes reach `main`. + +## Funnel documentation + +The canonical Funnel reference pages live in the `website/content/docs/` +directory on [Funnel's `develop` branch](https://github.com/calypr/funnel/tree/develop/website/content/docs). +A scheduled workflow checks that branch daily and opens or updates a pull request +with changed pages and generated navigation. Review and merge the pull request +to publish the update. The workflow also runs the sync tests and builds the site. + +The Funnel landing page, download page, and storage index remain CALYPR site +content. The installation script is copied from Funnel's `install.sh`. The sync +converts Funnel's Hugo shortcodes to Markdown compatible with this site and +rejects unknown shortcodes rather than publishing them verbatim. + +Three pages are temporarily held at their existing CALYPR versions while newer +site guidance is reconciled with Funnel: the GCP Batch, Kubernetes, and S3 +guides. Their expected upstream versions are recorded in +`docs/tools/funnel/.funnel-docs.json`. Once those exact versions reach `develop`, +the sync imports them and removes their exceptions automatically. If the +upstream content changes during review, update the expected hashes after +reconciling the differences. + +To run the sync locally against a Funnel checkout: + +```sh +python scripts/sync_funnel_docs.py /path/to/funnel --revision "$(git -C /path/to/funnel rev-parse HEAD)" +python -m unittest discover -s tests -v +make build ``` -## Where Navigation Lives - -Navigation is fully defined in [zensical.toml](./zensical.toml). - -If you want to: - -- rename a sidebar label -- move a section -- add a new page to the nav -- change the Tools grouping - -edit the `nav = [...]` structure in `zensical.toml`. - -There are no `.nav.yml` files anymore. If you add a page under `docs/` and do not add it to `zensical.toml`, it will not appear in the intended nav structure. - -## Source of Truth Rules - -### Repo-authored pages - -These live in `docs/` and should be edited directly there. - -Examples: - -- `docs/index.md` -- `docs/calypr/...` -- `docs/tools/index.md` -- `docs/tools/funnel/...` -- `docs/tools/grip/...` - -### Imported tool pages - -These are not authored in this repo anymore. - -Current imported repos: - -- `git-drs` -- `syfon` - -Their imported file mappings live in [scripts/prepare_docs.py](./scripts/prepare_docs.py) under `REPO_IMPORTS`. - -If you need to: - -- add a new imported page -- stop importing a page -- change where an imported page lands under `tools/...` - -update `REPO_IMPORTS` and usually also update `zensical.toml`. - -Do not hand-edit imported pages under `.generated/docs`. They are build artifacts and will be replaced on the next `make prepare`. - -## How Upstream Imports Work - -The importer prefers local sibling repos by default: - -- `../git-drs` -- `../syfon` - -So on a typical local machine, the imported content comes from whatever is currently checked out in those repos. - -If local sibling repos are unavailable, the importer falls back to GitHub and picks the first available branch from [scripts/branch_config.json](./scripts/branch_config.json). - -Current branch preference order: - -- `development` -- `develop` -- `main` -- `master` - -The last import decision is written to `.generated/imports.json`. - -Environment overrides: - -- `CALYPR_IMPORT_PREFER_LOCAL=0`: force remote import mode -- `CALYPR_IMPORT_GIT_DRS_DIR=/path/to/git-drs`: override local `git-drs` source -- `CALYPR_IMPORT_SYFON_DIR=/path/to/syfon`: override local `syfon` source - -## When You Add or Change Pages - -### Adding a repo-authored page - -1. Create the file under `docs/` -2. Add it to `zensical.toml` -3. Run `make serve` or `make build` - -### Adding an imported tool page - -1. Add the file mapping in `scripts/prepare_docs.py` -2. Add the page to `zensical.toml` -3. If the imported markdown links break after relocation, add a targeted rewrite in `MARKDOWN_REWRITES` -4. Run `make prepare` -5. Run `make build` - -## Styling and Overrides - -Site-level assets still come from the repo: - -- custom CSS: `docs/stylesheets/extra.css` -- custom JS: `docs/javascripts/nav-open-level1.js` -- theme overrides: `overrides/` - -Those are copied into `.generated/docs` during preparation and then used by Zensical. - -## Deployment Behavior - -Deployment uses the same pipeline as local builds: - -- GitHub Pages runs `python scripts/prepare_docs.py` and then `zensical build --clean` -- Netlify does the same - -That means deployment now depends on the import step succeeding. In CI, the importer will usually use remote GitHub repos rather than local sibling repos. - -## Files You Should Usually Ignore - -- `.generated/` -- `site/` -- `.cache/` - -These are build/runtime artifacts. - -## Known Caveat - -Zensical currently reports some warnings from imported and legacy docs content, especially around bracketed literals like: - -- `[SIGNED]` -- `[0]` -- `[]string` -- `map[string]string` +Generated Funnel pages and the marked Funnel navigation block should be changed +in the Funnel repository, not edited manually here. The sync branch is reused +while an update PR is open. If the Funnel revision is unchanged, no PR is created. -Those are content/rendering warnings, not a sign that the Zensical migration is broken. +The workflow needs both the organization policy and the repository setting that +allow GitHub Actions to create pull requests, plus `contents: write` and +`pull-requests: write` job permissions. +GitHub places PR workflows created with `GITHUB_TOKEN` in an approval-required +state. The sync workflow runs its own tests and site build before creating the +PR; a repository writer can approve the PR workflows separately. diff --git a/README.md b/README.md index 9fc1098..d23c89d 100644 --- a/README.md +++ b/README.md @@ -15,16 +15,14 @@ make serve The local build contract is: -- `make prepare` stages repo-authored docs plus imported upstream tool docs into `.generated/docs` - `make build` runs a clean Zensical build -- `make serve` stages the docs and starts the local Zensical server +- `make serve` starts the local Zensical server For repo maintenance details, navigation ownership, and import behavior, see [DEVELOPING.md](./DEVELOPING.md). If you prefer the raw commands: ```sh -python scripts/prepare_docs.py zensical serve ``` diff --git a/docs/tools/funnel/.funnel-docs.json b/docs/tools/funnel/.funnel-docs.json new file mode 100644 index 0000000..7c6f043 --- /dev/null +++ b/docs/tools/funnel/.funnel-docs.json @@ -0,0 +1,11 @@ +{ + "source": "https://github.com/calypr/funnel", + "branch": "develop", + "revision": "a3be3b208e0f6202bc2a33d1946fb56a8e850431", + "files": [], + "pending_reconciliation": { + "compute/gcp-batch.md": "a1ad1e72c4b7b8befd4ec98814855aa43fb48f8b76b6b12f9147ad64802f4d79", + "compute/kubernetes.md": "456c6cbed9d9d1687b717dde7e647daea896fd96b03662ec4022d862930ca400", + "storage/s3.md": "7624b293ea197bc68dfb01d06227b2c2da3b1ccd16762d2c00b3abfb01b4cf5b" + } +} diff --git a/scripts/sync_funnel_docs.py b/scripts/sync_funnel_docs.py new file mode 100644 index 0000000..d8f5c98 --- /dev/null +++ b/scripts/sync_funnel_docs.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Copy Funnel's reviewed develop docs into the CALYPR website checkout.""" +import argparse +import hashlib +import json +import re +from pathlib import Path + +SITE = Path(__file__).resolve().parents[1] +DEST = SITE / 'docs/tools/funnel/docs' +MANIFEST = SITE / 'docs/tools/funnel/.funnel-docs.json' +NAV = SITE / 'zensical.toml' +GROUPS = [('Storage', 'storage'), ('Compute', 'compute'), ('Databases', 'databases'), + ('Security', 'security'), ('Integrations', 'integrations'), ('Events', 'events'), + ('Metrics', 'metrics'), ('Development', 'development')] +TEMPLATES = {'htcondor': ('htcondor', 'HTCondor'), 'gridengine': ('gridengine', 'GridEngine'), + 'pbs': ('pbs', 'PBS'), 'slurm': ('slurm', 'Slurm')} + +def convert(text, upstream): + text = text.replace('https://ohsu-comp-bio.github.io/funnel/install.sh', 'https://calypr.org/tools/funnel/install.sh') + text = text.replace('src="/img/k8s-pvc.png"', 'src="../k8s-pvc.png"') + text = re.sub(r'\{\{< details title="([^"]*)" >\}\}', r'
\1\n', text) + text = text.replace('{{< /details >}}', '
') + converted_template = False + for shortcode, (compute, config) in TEMPLATES.items(): + token = '{{< ' + shortcode + '-template >}}' + if token in text: + converted_template = True + template = (upstream / 'website/static/funnel-config-examples' / (shortcode + '-template.txt')).read_text() + replacement = 'Compute: ' + compute + '\n\n' + config + ':\n Template: |\n' + ''.join(' ' + line + '\n' for line in template.splitlines()) + text = text.replace(token, replacement.rstrip()) + if re.search(r'\{\{[<%]', text): + raise ValueError('Unconverted Hugo shortcode; extend the sync before importing this page') + if converted_template: + text = re.sub(r'(?m)^[ \t]+$', '', text) + text = text.replace('as an argument \n', 'as an argument\n') + return text + +def title(path): + match = re.search(r'^title:\s*["\']?(.+?)["\']?\s*$', path.read_text(), re.MULTILINE) + return match.group(1) if match else path.stem.replace('-', ' ').title() + +def sync(upstream, revision): + source = upstream / 'website/content/docs' + files = [p for p in source.rglob('*.md') if p.parent != source or p.name == 'tasks.md'] + unknown = {p.relative_to(source).parts[0] for p in files if p.parent != source} - {folder for _, folder in GROUPS} + if unknown: + raise ValueError('Unknown Funnel documentation sections: ' + ', '.join(sorted(unknown))) + if not files or not (source / 'tasks.md').is_file(): + raise ValueError('Not a Funnel source checkout: website/content/docs/tasks.md is missing') + old = json.loads(MANIFEST.read_text()) if MANIFEST.exists() else {'files': []} + pending = old.get('pending_reconciliation', {}).copy() + new = sorted(p.relative_to(source).as_posix() for p in files) + missing_pending = set(pending) - set(new) + if missing_pending: + raise ValueError('Reconciliation pages missing upstream: ' + ', '.join(sorted(missing_pending))) + for name in old['files']: + p = Path(name) + if p.is_absolute() or '..' in p.parts: + raise ValueError('Invalid path in sync manifest') + if name not in new: + (DEST / p).unlink(missing_ok=True) + for path in files: + name = path.relative_to(source).as_posix() + if name in pending: + if hashlib.sha256(path.read_bytes()).hexdigest() != pending[name]: + continue + del pending[name] + target = DEST / path.relative_to(source) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(convert(path.read_text(), upstream)) + image = upstream / 'website/static/img/k8s-pvc.png' + if image.is_file(): + (DEST / 'compute/k8s-pvc.png').write_bytes(image.read_bytes()) + # This public installer is also distributed directly by the CALYPR site. + (SITE / 'docs/tools/funnel/install.sh').write_bytes((upstream / 'install.sh').read_bytes()) + + nav = [' # BEGIN GENERATED FUNNEL NAV', ' { "Funnel" = [', + ' "tools/funnel/index.md",', ' { "Overview" = "tools/funnel/docs/docs.md" },', + ' { "Download" = "tools/funnel/docs/download.md" },', + ' { "Tasks" = "tools/funnel/docs/tasks.md" },'] + for label, folder in GROUPS: + entries = sorted((p for p in files if p.relative_to(source).parts[0] == folder), key=lambda p: (title(p).lower(), p.name)) + if not entries: + continue + nav.append(' { ' + json.dumps(label) + ' = [') + if folder == 'storage': + nav.append(' "tools/funnel/docs/storage/index.md",') + for p in entries: + nav.append(' { ' + json.dumps(title(p), ensure_ascii=False) + ' = ' + json.dumps('tools/funnel/docs/' + p.relative_to(source).as_posix()) + ' },') + nav.append(' ] },') + nav += [' ] },', ' # END GENERATED FUNNEL NAV'] + config = NAV.read_text() + begin, end = ' # BEGIN GENERATED FUNNEL NAV', ' # END GENERATED FUNNEL NAV' + if begin not in config or end not in config: + raise ValueError('Missing Funnel navigation markers') + NAV.write_text(config[:config.index(begin)] + '\n'.join(nav) + config[config.index(end)+len(end):]) + MANIFEST.write_text(json.dumps({'source': 'https://github.com/calypr/funnel', 'branch': 'develop', 'revision': revision, 'files': new, 'pending_reconciliation': pending}, indent=2) + '\n') + print(f'Synced {len(new)} Funnel pages from {revision}') + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('upstream', type=Path) + parser.add_argument('--revision', required=True) + args = parser.parse_args() + if not re.fullmatch(r'[a-f0-9]{40}', args.revision): + parser.error('--revision must be a full Git commit SHA') + sync(args.upstream.resolve(), args.revision) diff --git a/tests/test_sync_funnel_docs.py b/tests/test_sync_funnel_docs.py new file mode 100644 index 0000000..cd36c60 --- /dev/null +++ b/tests/test_sync_funnel_docs.py @@ -0,0 +1,79 @@ +import hashlib +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location('sync_funnel_docs', Path(__file__).resolve().parents[1] / 'scripts/sync_funnel_docs.py') +sync = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(sync) +REV = 'a' * 40 + +class SyncTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + root = Path(self.temp.name) + self.site, self.upstream = root / 'site', root / 'upstream' + self.source = self.upstream / 'website/content/docs' + (self.source / 'compute').mkdir(parents=True) + (self.site / 'docs/tools/funnel/docs').mkdir(parents=True) + (self.upstream / 'install.sh').write_text('#!/bin/sh\n') + (self.source / 'tasks.md').write_text('---\ntitle: Tasks\n---\n') + (self.source / 'compute/example.md').write_text('---\ntitle: Example\n---\nnew\n') + self.nav = self.site / 'zensical.toml' + self.nav.write_text(' # BEGIN GENERATED FUNNEL NAV\n # END GENERATED FUNNEL NAV\n') + sync.SITE = self.site + sync.DEST = self.site / 'docs/tools/funnel/docs' + sync.MANIFEST = self.site / 'docs/tools/funnel/.funnel-docs.json' + sync.NAV = self.nav + + def tearDown(self): + self.temp.cleanup() + + def test_sync_is_repeatable_and_updates_navigation_and_removed_pages(self): + sync.sync(self.upstream, REV) + page = sync.DEST / 'compute/example.md' + self.assertIn('new', page.read_text()) + self.assertIn('tools/funnel/docs/compute/example.md', self.nav.read_text()) + old_nav, old_manifest = self.nav.read_text(), sync.MANIFEST.read_text() + sync.sync(self.upstream, REV) + self.assertEqual(old_nav, self.nav.read_text()) + self.assertEqual(old_manifest, sync.MANIFEST.read_text()) + (self.source / 'compute/example.md').unlink() + sync.sync(self.upstream, REV) + self.assertFalse(page.exists()) + self.assertNotIn('tools/funnel/docs/compute/example.md', self.nav.read_text()) + + def test_preserves_live_page_until_reconciled_upstream_arrives(self): + page = sync.DEST / 'compute/example.md' + page.parent.mkdir(parents=True) + page.write_text('live content') + expected = b'---\ntitle: Example\n---\nreconciled\n' + sync.MANIFEST.write_text(json.dumps({'files': [], 'pending_reconciliation': {'compute/example.md': hashlib.sha256(expected).hexdigest()}})) + sync.sync(self.upstream, REV) + self.assertEqual(page.read_text(), 'live content') + (self.source / 'compute/example.md').write_bytes(expected) + sync.sync(self.upstream, REV) + self.assertEqual(page.read_bytes(), expected) + self.assertEqual(json.loads(sync.MANIFEST.read_text())['pending_reconciliation'], {}) + (self.source / 'compute/example.md').write_text('later change') + sync.sync(self.upstream, REV) + self.assertEqual(page.read_text(), 'later change') + + def test_does_not_delete_a_pending_page_when_upstream_removes_it(self): + page = sync.DEST / 'compute/example.md' + page.parent.mkdir(parents=True) + page.write_text('live content') + sync.MANIFEST.write_text(json.dumps({'files': ['compute/example.md'], 'pending_reconciliation': {'compute/example.md': 'expected'}})) + (self.source / 'compute/example.md').unlink() + with self.assertRaisesRegex(ValueError, 'Reconciliation pages missing upstream'): + sync.sync(self.upstream, REV) + self.assertEqual(page.read_text(), 'live content') + + def test_fails_for_unknown_shortcodes(self): + with self.assertRaisesRegex(ValueError, 'Unconverted Hugo shortcode'): + sync.convert('{{< surprise >}}', self.upstream) + +if __name__ == '__main__': + unittest.main() diff --git a/zensical.toml b/zensical.toml index 4651955..8c148d2 100644 --- a/zensical.toml +++ b/zensical.toml @@ -93,6 +93,7 @@ nav = [ ] }, ] }, { "Workflow Execution" = [ + # BEGIN GENERATED FUNNEL NAV { "Funnel" = [ "tools/funnel/index.md", { "Overview" = "tools/funnel/docs/docs.md" }, @@ -143,6 +144,7 @@ nav = [ { "Funnel Developers" = "tools/funnel/docs/development/developers.md" }, ] }, ] }, + # END GENERATED FUNNEL NAV ] }, { "Graph Database" = [ { "GRIP" = [