Release to PyPI from a pushed vX.Y.Z tag, through CI

.github/workflows/ci.yml

@@ -4,6 +4,8 @@ on:
   push:
     branches: [main]
   pull_request:
+  # release.yml runs this as its gate, so that a release passes the same CI as a pull request.
+  workflow_call:
 
 jobs:
   # The Cypher spec's copies in tests/cypher are the tag tests/cypher/VERSION names, unchanged, and a later tag is

.github/workflows/release.yml

@@ -0,0 +1,161 @@
+# Releases drivine to PyPI from a tag. RELEASING.md describes the process.
+#
+# - A push of a tag vX.Y.Z publishes: the tag is checked, the full CI runs, the distributions are built and checked,
+#   and those exact files are uploaded to PyPI with trusted publishing.
+# - workflow_dispatch, and a pull request that changes the release machinery, are dry runs: everything but the
+#   publish, with the built files left as the run's artifact to inspect.
+name: Release
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - .github/workflows/release.yml
+      - RELEASING.md
+      - pyproject.toml
+      - scripts/smoke_wheel.py
+
+permissions:
+  contents: read
+
+concurrency:
+  group: release-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+      publish: ${{ steps.version.outputs.publish }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+      - name: Work out the version
+        id: version
+        env:
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        run: |
+          declared=$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')
+          if [[ "$GITHUB_EVENT_NAME" == "push" && "$GITHUB_REF" == refs/tags/* ]]; then
+            tag="$GITHUB_REF_NAME"
+            if [[ ! "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then
+              echo "::error::The tag $tag isn't vX.Y.Z (or vX.Y.Z-suffix). Delete it and tag the release commit vX.Y.Z."
+              exit 1
+            fi
+            version="${tag#v}"
+            if [[ "$version" != "$declared" ]]; then
+              echo "::error::The tag $tag is version $version, but pyproject.toml's version is $declared. Tag the commit whose version is $version, or bump pyproject.toml in a release PR first."
+              exit 1
+            fi
+            if ! git merge-base --is-ancestor "$GITHUB_SHA" "origin/$DEFAULT_BRANCH"; then
+              echo "::error::The tag $tag is on $GITHUB_SHA, which isn't on $DEFAULT_BRANCH. Tag the release PR's merge commit on $DEFAULT_BRANCH."
+              exit 1
+            fi
+            publish=true
+          else
+            version="$declared"
+            publish=false
+          fi
+          echo "Version $version, publish: $publish"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+          echo "publish=$publish" >> "$GITHUB_OUTPUT"
+
+  # The full CI, as ci.yml runs it on a push or a pull request.
+  gate:
+    needs: check
+    uses: ./.github/workflows/ci.yml
+
+  build:
+    needs: [check, gate]
+    runs-on: ubuntu-latest
+    env:
+      VERSION: ${{ needs.check.outputs.version }}
+      PUBLISH: ${{ needs.check.outputs.publish }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        with:
+          python-version: "3.14"
+          enable-cache: false
+      # The sdist, then the wheel built from it, into dist/.
+      - name: Build
+        run: uv build --no-sources
+      # PyPI refuses a version it already has, so a tag of one fails here, before anything is uploaded. The wheel's
+      # name holds the version as PEP 440 normalizes it, as PyPI has it: 0.3.0rc1 for 0.3.0-rc.1. When PyPI can't be
+      # asked, the run goes on with a warning: PyPI would refuse the upload of a version it has, overwriting nothing.
+      - name: PyPI doesn't have this version yet
+        run: |
+          wheels=(dist/*.whl)
+          name=${wheels[0]#dist/drivine-}
+          normalized=${name%%-*}
+          url="https://pypi.org/pypi/drivine/$normalized/json"
+          status=$(curl -sL --retry 3 -o /dev/null -w '%{http_code}' "$url" || true)
+          if [[ "$status" == 200 ]]; then
+            if [[ "$PUBLISH" == true ]]; then
+              echo "::error::PyPI already has drivine $normalized."
+              exit 1
+            fi
+            echo "::warning::PyPI already has drivine $normalized. This dry run goes on, but a tag of it would fail."
+          elif [[ "$status" != 404 ]]; then
+            echo "::warning::Couldn't ask PyPI whether it has drivine $normalized (HTTP status $status). Going on."
+          fi
+      - name: Check the distributions
+        run: uvx twine check --strict dist/*
+      # The wheel, installed into a clean environment on the oldest and the newest Python drivine supports, imports
+      # and writes typed references with python -m drivine.stubs.
+      - name: Smoke test the wheel on Python 3.11
+        run: uv run --no-project --python 3.11 python scripts/smoke_wheel.py dist/*.whl
+      - name: Smoke test the wheel on Python 3.14
+        run: uv run --no-project --python 3.14 python scripts/smoke_wheel.py dist/*.whl
+      - name: List the distributions
+        run: |
+          echo "drivine $VERSION"
+          ls -l dist
+          sha256sum dist/*
+          unzip -l dist/*.whl
+          tar -tzf dist/*.tar.gz
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+  publish:
+    needs: [check, build]
+    if: needs.check.outputs.publish == 'true'
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+      url: https://pypi.org/project/drivine/${{ needs.check.outputs.version }}/
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: dist
+          path: dist/
+      # Trusted publishing: PyPI trusts this workflow, in the release environment, through OIDC. No token is held.
+      - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3
+        with:
+          packages-dir: dist/
+
+  github-release:
+    needs: [check, publish]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Create the GitHub release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ github.ref_name }}
+          VERSION: ${{ needs.check.outputs.version }}
+        run: |
+          prerelease=()
+          if [[ "$VERSION" == *-* ]]; then prerelease=(--prerelease); fi
+          gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --generate-notes --verify-tag "${prerelease[@]}"

RELEASING.md

@@ -0,0 +1,75 @@
+# Releasing
+
+drivine-python, drivine4j and drivine are released the same way, from a tag, by CI. Each repository has this file,
+differing only in its publish step.
+
+## The process
+
+1. **Release PR.** A PR bumps the version (build.gradle / package.json / pyproject.toml) and the changelog, if the
+   repo has one. It is reviewed and passes CI like any other PR.
+2. **Tag the merge commit.** Merging the release PR is the release point. The maintainer then tags the merge commit
+   on the default branch `vX.Y.Z` and pushes the tag.
+3. **CI publishes.** The tag push runs `.github/workflows/release.yml`, which checks out the tag, runs the full gate,
+   builds, checks, and publishes. Nothing is built from a laptop.
+4. **Credentials.** Repository secrets or trusted publishing (OIDC), held in a GitHub environment named `release`.
+
+For step 2:
+
+```
+git switch main && git pull
+git tag vX.Y.Z
+git push origin vX.Y.Z
+```
+
+## What release.yml does
+
+- **`check`** works out the version: the tag without its `v` on a tag push, or the build file's version otherwise.
+  On a tag push it fails, saying why, unless the tag is `vX.Y.Z` (or `vX.Y.Z-suffix`), its version equals the build
+  file's, and the tagged commit is on the default branch.
+- **`gate`** runs the repository's full CI, `ci.yml`, unchanged.
+- **`build`** builds the release artifacts from the checked-out ref, checks them, runs a smoke test, and uploads them
+  as the run's artifact.
+- **`publish`** runs on a tag push alone, in the `release` environment. It publishes the exact files `build` made.
+- **`github-release`** then creates the GitHub release for the tag, with generated notes, marked a prerelease when
+  the version has a `-suffix`.
+
+A tag that fails `check` publishes nothing. Delete it (`git push origin :refs/tags/vX.Y.Z`), fix what the message
+says, and tag again.
+
+A **dry run** is everything but `publish` and `github-release`. Run it from the Actions tab (Release, Run workflow,
+on any branch or tag), and it runs by itself on a pull request that changes the release machinery. A tag push is the
+only thing that publishes.
+
+## drivine-python: PyPI
+
+- **Version:** `version` in `pyproject.toml`. The tag must be `v` and that version, character for character: `v0.3.0`
+  for `0.3.0`. There is no `__version__` to keep in step.
+- **Prereleases:** write the version in `pyproject.toml` as the tag has it, with a `-`: `0.3.0-rc.1`, tagged
+  `v0.3.0-rc.1`. PEP 440 normalizes it, so the distributions and PyPI call it `0.3.0rc1`, and pip installs it only
+  when asked (`pip install drivine==0.3.0rc1`, or `--pre`). `0.3.0rc1` in `pyproject.toml` can't be tagged: `check`
+  wants a `-` in the tag. The suffix must be one PEP 440 reads (`a`, `b`, `rc`, `alpha`, `beta`, `c`, `pre`, `dev`, with
+  a number), or the build fails before anything is published.
+- **Build:** `uv build --no-sources`, which writes the sdist and the wheel built from it to `dist/` (hatchling).
+- **Checks:** `build` fails a tag of a version PyPI already has, since PyPI refuses to take it again. Then
+  `twine check --strict` on both, then `scripts/smoke_wheel.py`, which installs the wheel into a clean
+  virtual environment on Python 3.11 and 3.14, imports `drivine`, and runs `python -m drivine.stubs` on a small
+  project's models.
+- **Publish:** `pypa/gh-action-pypi-publish` uploads the built `dist/` to [PyPI](https://pypi.org/project/drivine/)
+  with trusted publishing (OIDC). No token is held anywhere.
+- **Dry run:** leaves the `dist` artifact on the run: `drivine-X.Y.Z.tar.gz` and `drivine-X.Y.Z-py3-none-any.whl`.
+  A dry run of a version PyPI already has only warns.
+
+The same checks, locally, publishing nothing:
+
+```
+uv build --no-sources
+uvx twine check --strict dist/*
+python scripts/smoke_wheel.py dist/*.whl
+```
+
+### Settings, made once
+
+- **GitHub:** Settings, Environments, a `release` environment. It can carry required reviewers, and a deployment tag
+  rule `v*`.
+- **PyPI:** the `drivine` project's Publishing settings, a GitHub trusted publisher with owner `liberation-data`,
+  repository `drivine-python`, workflow `release.yml`, and environment `release`.

scripts/smoke_wheel.py

@@ -0,0 +1,99 @@
+"""
+Installs a built wheel into a clean virtual environment, outside the repository, and checks that it works: drivine
+imports, and `python -m drivine.stubs` writes the typed references of a small project's models. The release workflow
+runs it on the wheel it publishes:
+
+    python scripts/smoke_wheel.py dist/drivine-X.Y.Z-py3-none-any.whl
+
+It reads the standard library alone, so nothing is installed for it but the wheel, into the environment it makes.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import tempfile
+import venv
+from pathlib import Path
+
+MODELS = """\
+from typing import Annotated
+
+from drivine import GraphView, NodeFragment, NodeId, Relationship, RelationshipFragment, Root, TargetNode
+
+
+class Person(NodeFragment, labels=["Person"]):
+    uuid: Annotated[str, NodeId()]
+    name: str
+
+
+class Organization(NodeFragment, labels=["Organization"]):
+    uuid: Annotated[str, NodeId()]
+    name: str
+
+
+class WorkHistory(RelationshipFragment):
+    role: str
+    organization: Annotated[Organization, TargetNode()]
+
+
+class PersonCareer(GraphView):
+    person: Annotated[Person, Root()]
+    employment_history: Annotated[list[WorkHistory], Relationship("WORKS_FOR")] = []
+"""
+
+# drivine is imported from the environment's site-packages, from the wheel, and not from a checkout.
+IMPORTS = (
+    "import sys, drivine; assert drivine.__file__.startswith(sys.prefix), drivine.__file__; print(drivine.__file__)"
+)
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) != 1 or not argv[0].endswith(".whl"):
+        print("Usage: python scripts/smoke_wheel.py <the wheel>", file=sys.stderr)
+        return 2
+    wheel = Path(argv[0]).resolve()
+    with tempfile.TemporaryDirectory(prefix="drivine-smoke-") as root:
+        home = Path(root)
+        venv.create(home / "venv", with_pip=True)
+        python = home / "venv" / ("Scripts" if sys.platform == "win32" else "bin") / "python"
+        project = home / "project"
+        (project / "smoke").mkdir(parents=True)
+        (project / "pyproject.toml").write_text('[project]\nname = "smoke"\nversion = "0"\n')
+        (project / "smoke" / "__init__.py").write_text("")
+        (project / "smoke" / "models.py").write_text(MODELS)
+
+        # Nothing from the repository or the caller's environment is on the path: drivine comes from the wheel alone.
+        env = {key: value for key, value in os.environ.items() if key not in ("PYTHONPATH", "VIRTUAL_ENV")}
+        env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
+
+        def run(*args: str) -> None:
+            print("+", " ".join(args), flush=True)
+            subprocess.run(args, cwd=project, env=env, check=True)
+
+        run(str(python), "-m", "pip", "install", "--quiet", str(wheel))
+        run(str(python), "-c", IMPORTS)
+        run(str(python), "-m", "drivine.stubs", "smoke.models")
+        written = [
+            project / "smoke" / "models_refs.py",
+            project / "smoke" / "models_refs.pyi",
+            project / ".drivine" / "model.json",
+        ]
+        missing = [str(path.relative_to(project)) for path in written if not path.is_file()]
+        if missing:
+            print(f"python -m drivine.stubs didn't write {', '.join(missing)}.", file=sys.stderr)
+            return 1
+        run(str(python), "-m", "drivine.stubs", "smoke.models", "--check")
+        run(
+            str(python),
+            "-c",
+            "from smoke.models_refs import refs; from smoke.models import PersonCareer; "
+            "print(refs(PersonCareer).person.name.eq('Ada'))",
+        )
+    print(f"{wheel.name}: installs, imports, and writes typed references.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))