From 882c86c3d30a4a46e0bd2dae369fe467dd395bd1 Mon Sep 17 00:00:00 2001 From: jasper blues Date: Wed, 28 Oct 2026 05:42:57 +1000 Subject: [PATCH 1/3] Release to PyPI from a pushed vX.Y.Z tag, through CI. The tag is checked against pyproject.toml's version and the default branch, the full CI runs, and the sdist and wheel are built, checked with twine and a smoke test of the wheel, and published with trusted publishing. A manual run, or a pull request that changes the release machinery, is a dry run that leaves the built files to inspect. RELEASING.md describes the process. Closes #158. --- .github/workflows/ci.yml | 2 + .github/workflows/release.yml | 136 ++++++++++++++++++++++++++++++++++ RELEASING.md | 64 ++++++++++++++++ scripts/smoke_wheel.py | 99 +++++++++++++++++++++++++ 4 files changed, 301 insertions(+) create mode 100644 .github/workflows/release.yml create mode 100644 RELEASING.md create mode 100644 scripts/smoke_wheel.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d133cd9..3b6859a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,8 @@ on: push: branches: [main] pull_request: + # release.yml runs this as its gate, so that a release passes the same CI as a pull request. + workflow_call: jobs: # The Cypher spec's copies in tests/cypher are the tag tests/cypher/VERSION names, unchanged, and a later tag is diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ba9463e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,136 @@ +# Releases drivine to PyPI from a tag. RELEASING.md describes the process. +# +# - A push of a tag vX.Y.Z publishes: the tag is checked, the full CI runs, the distributions are built and checked, +# and those exact files are uploaded to PyPI with trusted publishing. +# - workflow_dispatch, and a pull request that changes the release machinery, are dry runs: everything but the +# publish, with the built files left as the run's artifact to inspect. +name: Release + +on: + push: + tags: ["v*"] + workflow_dispatch: + pull_request: + paths: + - .github/workflows/release.yml + - RELEASING.md + - pyproject.toml + - scripts/smoke_wheel.py + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + check: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + publish: ${{ steps.version.outputs.publish }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: Work out the version + id: version + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + declared=$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])') + if [[ "$GITHUB_EVENT_NAME" == "push" && "$GITHUB_REF" == refs/tags/* ]]; then + tag="$GITHUB_REF_NAME" + if [[ ! "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo "::error::The tag $tag isn't vX.Y.Z (or vX.Y.Z-suffix). Delete it and tag the release commit vX.Y.Z." + exit 1 + fi + version="${tag#v}" + if [[ "$version" != "$declared" ]]; then + echo "::error::The tag $tag is version $version, but pyproject.toml's version is $declared. Tag the commit whose version is $version, or bump pyproject.toml in a release PR first." + exit 1 + fi + if ! git merge-base --is-ancestor "$GITHUB_SHA" "origin/$DEFAULT_BRANCH"; then + echo "::error::The tag $tag is on $GITHUB_SHA, which isn't on $DEFAULT_BRANCH. Tag the release PR's merge commit on $DEFAULT_BRANCH." + exit 1 + fi + publish=true + else + version="$declared" + publish=false + fi + echo "Version $version, publish: $publish" + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "publish=$publish" >> "$GITHUB_OUTPUT" + + # The full CI, as ci.yml runs it on a push or a pull request. + gate: + uses: ./.github/workflows/ci.yml + + build: + needs: [check, gate] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + with: + python-version: "3.14" + enable-cache: false + # The sdist, then the wheel built from it, into dist/. + - name: Build + run: uv build --no-sources + - name: Check the distributions + run: uvx twine check --strict dist/* + # The wheel, installed into a clean environment on the oldest and the newest Python drivine supports, imports + # and writes typed references with python -m drivine.stubs. + - name: Smoke test the wheel on Python 3.11 + run: uv run --no-project --python 3.11 python scripts/smoke_wheel.py dist/*.whl + - name: Smoke test the wheel on Python 3.14 + run: uv run --no-project --python 3.14 python scripts/smoke_wheel.py dist/*.whl + - name: List the distributions + env: + VERSION: ${{ needs.check.outputs.version }} + run: | + echo "drivine $VERSION" + ls -l dist + sha256sum dist/* + unzip -l dist/*.whl + tar -tzf dist/*.tar.gz + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dist + path: dist/ + if-no-files-found: error + + publish: + needs: [check, build] + if: needs.check.outputs.publish == 'true' + runs-on: ubuntu-latest + environment: + name: release + url: https://pypi.org/project/drivine/${{ needs.check.outputs.version }}/ + permissions: + contents: read + id-token: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dist + path: dist/ + # Trusted publishing: PyPI trusts this workflow, in the release environment, through OIDC. No token is held. + - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + packages-dir: dist/ + + github-release: + needs: [check, publish] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Create the GitHub release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --generate-notes --verify-tag diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..1c5945e --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,64 @@ +# Releasing + +drivine-python, drivine4j and drivine are released the same way, from a tag, by CI. Each repository has this file, +differing only in its publish step. + +## The process + +1. **Release PR.** A PR bumps the version (build.gradle / package.json / pyproject.toml) and the changelog, if the + repo has one. It is reviewed and passes CI like any other PR. +2. **Tag the merge commit.** Merging the release PR is the release point. The maintainer then tags the merge commit + on the default branch `vX.Y.Z` and pushes the tag. +3. **CI publishes.** The tag push runs `.github/workflows/release.yml`, which checks out the tag, runs the full gate, + builds, checks, and publishes. Nothing is built from a laptop. +4. **Credentials.** Repository secrets or trusted publishing (OIDC), held in a GitHub environment named `release`. + +For step 2: + +``` +git switch main && git pull +git tag vX.Y.Z +git push origin vX.Y.Z +``` + +## What release.yml does + +- **`check`** works out the version: the tag without its `v` on a tag push, or the build file's version otherwise. + On a tag push it fails, saying why, unless the tag is `vX.Y.Z` (or `vX.Y.Z-suffix`), its version equals the build + file's, and the tagged commit is on the default branch. +- **`gate`** runs the repository's full CI, `ci.yml`, unchanged. +- **`build`** builds the release artifacts from the checked-out ref, checks them, runs a smoke test, and uploads them + as the run's artifact. +- **`publish`** runs on a tag push alone, in the `release` environment. It publishes the exact files `build` made. +- **`github-release`** then creates the GitHub release for the tag, with generated notes. + +A **dry run** is everything but `publish` and `github-release`. Run it from the Actions tab (Release, Run workflow, +on any branch or tag), and it runs by itself on a pull request that changes the release machinery. A tag push is the +only thing that publishes. + +## drivine-python: PyPI + +- **Version:** `version` in `pyproject.toml`. The tag must be `v` and that version, character for character: `v0.3.0` + for `0.3.0`. There is no `__version__` to keep in step. +- **Build:** `uv build --no-sources`, which writes the sdist and the wheel built from it to `dist/` (hatchling). +- **Checks:** `twine check --strict` on both, then `scripts/smoke_wheel.py`, which installs the wheel into a clean + virtual environment on Python 3.11 and 3.14, imports `drivine`, and runs `python -m drivine.stubs` on a small + project's models. +- **Publish:** `pypa/gh-action-pypi-publish` uploads the built `dist/` to [PyPI](https://pypi.org/project/drivine/) + with trusted publishing (OIDC). No token is held anywhere. +- **Dry run:** leaves the `dist` artifact on the run: `drivine-X.Y.Z.tar.gz` and `drivine-X.Y.Z-py3-none-any.whl`. + +The same checks, locally, publishing nothing: + +``` +uv build --no-sources +uvx twine check --strict dist/* +python scripts/smoke_wheel.py dist/*.whl +``` + +### Settings, made once + +- **GitHub:** Settings, Environments, a `release` environment. It can carry required reviewers, and a deployment tag + rule `v*`. +- **PyPI:** the `drivine` project's Publishing settings, a GitHub trusted publisher with owner `liberation-data`, + repository `drivine-python`, workflow `release.yml`, and environment `release`. diff --git a/scripts/smoke_wheel.py b/scripts/smoke_wheel.py new file mode 100644 index 0000000..1c8117b --- /dev/null +++ b/scripts/smoke_wheel.py @@ -0,0 +1,99 @@ +""" +Installs a built wheel into a clean virtual environment, outside the repository, and checks that it works: drivine +imports, and `python -m drivine.stubs` writes the typed references of a small project's models. The release workflow +runs it on the wheel it publishes: + + python scripts/smoke_wheel.py dist/drivine-X.Y.Z-py3-none-any.whl + +It reads the standard library alone, so nothing is installed for it but the wheel, into the environment it makes. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import venv +from pathlib import Path + +MODELS = """\ +from typing import Annotated + +from drivine import GraphView, NodeFragment, NodeId, Relationship, RelationshipFragment, Root, TargetNode + + +class Person(NodeFragment, labels=["Person"]): + uuid: Annotated[str, NodeId()] + name: str + + +class Organization(NodeFragment, labels=["Organization"]): + uuid: Annotated[str, NodeId()] + name: str + + +class WorkHistory(RelationshipFragment): + role: str + organization: Annotated[Organization, TargetNode()] + + +class PersonCareer(GraphView): + person: Annotated[Person, Root()] + employment_history: Annotated[list[WorkHistory], Relationship("WORKS_FOR")] = [] +""" + +# drivine is imported from the environment's site-packages, from the wheel, and not from a checkout. +IMPORTS = ( + "import sys, drivine; assert drivine.__file__.startswith(sys.prefix), drivine.__file__; print(drivine.__file__)" +) + + +def main(argv: list[str]) -> int: + if len(argv) != 1 or not argv[0].endswith(".whl"): + print("Usage: python scripts/smoke_wheel.py ", file=sys.stderr) + return 2 + wheel = Path(argv[0]).resolve() + with tempfile.TemporaryDirectory(prefix="drivine-smoke-") as root: + home = Path(root) + venv.create(home / "venv", with_pip=True) + python = home / "venv" / ("Scripts" if sys.platform == "win32" else "bin") / "python" + project = home / "project" + (project / "smoke").mkdir(parents=True) + (project / "pyproject.toml").write_text('[project]\nname = "smoke"\nversion = "0"\n') + (project / "smoke" / "__init__.py").write_text("") + (project / "smoke" / "models.py").write_text(MODELS) + + # Nothing from the repository or the caller's environment is on the path: drivine comes from the wheel alone. + env = {key: value for key, value in os.environ.items() if key not in ("PYTHONPATH", "VIRTUAL_ENV")} + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + + def run(*args: str) -> None: + print("+", " ".join(args), flush=True) + subprocess.run(args, cwd=project, env=env, check=True) + + run(str(python), "-m", "pip", "install", "--quiet", str(wheel)) + run(str(python), "-c", IMPORTS) + run(str(python), "-m", "drivine.stubs", "smoke.models") + written = [ + project / "smoke" / "models_refs.py", + project / "smoke" / "models_refs.pyi", + project / ".drivine" / "model.json", + ] + missing = [str(path.relative_to(project)) for path in written if not path.is_file()] + if missing: + print(f"python -m drivine.stubs didn't write {', '.join(missing)}.", file=sys.stderr) + return 1 + run(str(python), "-m", "drivine.stubs", "smoke.models", "--check") + run( + str(python), + "-c", + "from smoke.models_refs import refs; from smoke.models import PersonCareer; " + "print(refs(PersonCareer).person.name.eq('Ada'))", + ) + print(f"{wheel.name}: installs, imports, and writes typed references.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) From f7902af221782e46f1437773293d02b33cccd88b Mon Sep 17 00:00:00 2001 From: jasper blues Date: Wed, 28 Oct 2026 05:49:28 +1000 Subject: [PATCH 2/3] Fail a tag PyPI already has, mark a prerelease as one, and gate CI on the tag check --- .github/workflows/release.yml | 27 ++++++++++++++++++++++++--- RELEASING.md | 15 +++++++++++++-- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ba9463e..72f79d2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,11 +66,15 @@ jobs: # The full CI, as ci.yml runs it on a push or a pull request. gate: + needs: check uses: ./.github/workflows/ci.yml build: needs: [check, gate] runs-on: ubuntu-latest + env: + VERSION: ${{ needs.check.outputs.version }} + PUBLISH: ${{ needs.check.outputs.publish }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 @@ -80,6 +84,21 @@ jobs: # The sdist, then the wheel built from it, into dist/. - name: Build run: uv build --no-sources + # PyPI refuses a version it already has, so a tag of one fails here, before anything is uploaded. The wheel's + # name holds the version as PEP 440 normalizes it, as PyPI has it: 0.3.0rc1 for 0.3.0-rc.1. + - name: PyPI doesn't have this version yet + run: | + wheels=(dist/*.whl) + name=${wheels[0]#dist/drivine-} + normalized=${name%%-*} + status=$(curl -sL -o /dev/null -w '%{http_code}' "https://pypi.org/pypi/drivine/$normalized/json") + if [[ "$status" == 200 ]]; then + if [[ "$PUBLISH" == true ]]; then + echo "::error::PyPI already has drivine $normalized." + exit 1 + fi + echo "::warning::PyPI already has drivine $normalized. This dry run goes on, but a tag of it would fail." + fi - name: Check the distributions run: uvx twine check --strict dist/* # The wheel, installed into a clean environment on the oldest and the newest Python drivine supports, imports @@ -89,8 +108,6 @@ jobs: - name: Smoke test the wheel on Python 3.14 run: uv run --no-project --python 3.14 python scripts/smoke_wheel.py dist/*.whl - name: List the distributions - env: - VERSION: ${{ needs.check.outputs.version }} run: | echo "drivine $VERSION" ls -l dist @@ -133,4 +150,8 @@ jobs: env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --generate-notes --verify-tag + VERSION: ${{ needs.check.outputs.version }} + run: | + prerelease=() + if [[ "$VERSION" == *-* ]]; then prerelease=(--prerelease); fi + gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --generate-notes --verify-tag "${prerelease[@]}" diff --git a/RELEASING.md b/RELEASING.md index 1c5945e..e5b9633 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -30,7 +30,11 @@ git push origin vX.Y.Z - **`build`** builds the release artifacts from the checked-out ref, checks them, runs a smoke test, and uploads them as the run's artifact. - **`publish`** runs on a tag push alone, in the `release` environment. It publishes the exact files `build` made. -- **`github-release`** then creates the GitHub release for the tag, with generated notes. +- **`github-release`** then creates the GitHub release for the tag, with generated notes, marked a prerelease when + the version has a `-suffix`. + +A tag that fails `check` publishes nothing. Delete it (`git push origin :refs/tags/vX.Y.Z`), fix what the message +says, and tag again. A **dry run** is everything but `publish` and `github-release`. Run it from the Actions tab (Release, Run workflow, on any branch or tag), and it runs by itself on a pull request that changes the release machinery. A tag push is the @@ -40,13 +44,20 @@ only thing that publishes. - **Version:** `version` in `pyproject.toml`. The tag must be `v` and that version, character for character: `v0.3.0` for `0.3.0`. There is no `__version__` to keep in step. +- **Prereleases:** write the version in `pyproject.toml` as the tag has it, with a `-`: `0.3.0-rc.1`, tagged + `v0.3.0-rc.1`. PEP 440 normalizes it, so the distributions and PyPI call it `0.3.0rc1`, and pip installs it only + when asked (`pip install drivine==0.3.0rc1`, or `--pre`). `0.3.0rc1` in `pyproject.toml` can't be tagged: `check` + wants a `-` in the tag. The suffix must be one PEP 440 reads (`a`, `b`, `rc`, `alpha`, `beta`, `c`, `pre`, `dev`, with + a number), or the build fails before anything is published. - **Build:** `uv build --no-sources`, which writes the sdist and the wheel built from it to `dist/` (hatchling). -- **Checks:** `twine check --strict` on both, then `scripts/smoke_wheel.py`, which installs the wheel into a clean +- **Checks:** `build` fails a tag of a version PyPI already has, since PyPI refuses to take it again. Then + `twine check --strict` on both, then `scripts/smoke_wheel.py`, which installs the wheel into a clean virtual environment on Python 3.11 and 3.14, imports `drivine`, and runs `python -m drivine.stubs` on a small project's models. - **Publish:** `pypa/gh-action-pypi-publish` uploads the built `dist/` to [PyPI](https://pypi.org/project/drivine/) with trusted publishing (OIDC). No token is held anywhere. - **Dry run:** leaves the `dist` artifact on the run: `drivine-X.Y.Z.tar.gz` and `drivine-X.Y.Z-py3-none-any.whl`. + A dry run of a version PyPI already has only warns. The same checks, locally, publishing nothing: From 9f4cbd503625f0d8554c3402814a4f461d2a913b Mon Sep 17 00:00:00 2001 From: jasper blues Date: Wed, 28 Oct 2026 05:55:12 +1000 Subject: [PATCH 3/3] Carry on with a warning when PyPI can't be asked whether it has the version --- .github/workflows/release.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 72f79d2..2e0d303 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -85,19 +85,23 @@ jobs: - name: Build run: uv build --no-sources # PyPI refuses a version it already has, so a tag of one fails here, before anything is uploaded. The wheel's - # name holds the version as PEP 440 normalizes it, as PyPI has it: 0.3.0rc1 for 0.3.0-rc.1. + # name holds the version as PEP 440 normalizes it, as PyPI has it: 0.3.0rc1 for 0.3.0-rc.1. When PyPI can't be + # asked, the run goes on with a warning: PyPI would refuse the upload of a version it has, overwriting nothing. - name: PyPI doesn't have this version yet run: | wheels=(dist/*.whl) name=${wheels[0]#dist/drivine-} normalized=${name%%-*} - status=$(curl -sL -o /dev/null -w '%{http_code}' "https://pypi.org/pypi/drivine/$normalized/json") + url="https://pypi.org/pypi/drivine/$normalized/json" + status=$(curl -sL --retry 3 -o /dev/null -w '%{http_code}' "$url" || true) if [[ "$status" == 200 ]]; then if [[ "$PUBLISH" == true ]]; then echo "::error::PyPI already has drivine $normalized." exit 1 fi echo "::warning::PyPI already has drivine $normalized. This dry run goes on, but a tag of it would fail." + elif [[ "$status" != 404 ]]; then + echo "::warning::Couldn't ask PyPI whether it has drivine $normalized (HTTP status $status). Going on." fi - name: Check the distributions run: uvx twine check --strict dist/*