diff --git a/.github/scripts/release.py b/.github/scripts/release.py new file mode 100644 index 0000000..a32119c --- /dev/null +++ b/.github/scripts/release.py @@ -0,0 +1,209 @@ +"""Steps of the release workflow (.github/workflows/release.yml). + + release.py check VERSION + Stop unless VERSION can be released now: a normalized PEP 440 + version after the one in pyproject.toml, without a tag yet, with + news fragments, run from main. + release.py set-version VERSION + Set the version in pyproject.toml. + release.py back-to-development VERSION + Set the version in pyproject.toml to the development version after + VERSION (the way zest.releaser does) and print it. + release.py prerelease VERSION + Print "true" if VERSION is a pre-release (a, b or rc), else "false". + +The functions do not read files or run git, so that the tests can call them. +Needs the packaging distribution. +""" + +from packaging.version import InvalidVersion +from packaging.version import Version +from pathlib import Path + +import argparse +import os +import re +import subprocess +import sys + + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + + +PYPROJECT = Path("pyproject.toml") +MAIN = "refs/heads/main" + +_PROJECT_TABLE = re.compile(r"^\[project\][ \t]*$", re.MULTILINE) +_NEXT_TABLE = re.compile(r"^\[", re.MULTILINE) +_VERSION_LINE = re.compile(r'^(version[ \t]*=[ \t]*)"[^"\n]*"', re.MULTILINE) + + +def check_release( + version: str, + current: str, + tags: list[str], + names: list[str], + types: list[str], + ref: str, +) -> list[str]: + """Return the reasons why version cannot be released, or [].""" + errors = [] + if ref != MAIN: + errors.append(f"Run the release from main, not from {ref!r}") + shape = _version_errors(version) + if shape: + return errors + shape + if Version(version) <= Version(current): + errors.append(f"{version} is not after the version {current} in pyproject.toml") + if version in tags: + errors.append(f"The tag {version} exists already") + if not any(_is_fragment(name, types) for name in names): + errors.append("There are no news fragments in news/") + return errors + + +def _version_errors(version: str) -> list[str]: + try: + parsed = Version(version) + except InvalidVersion: + return [f"{version!r} is not a PEP 440 version"] + if str(parsed) != version: + return [f"{version!r} is not in the normalized form {str(parsed)!r}"] + if parsed.is_devrelease: + return [f"{version} is a development version"] + if parsed.local is not None: + return [f"{version} has a local part"] + if parsed.epoch: + return [f"{version} has an epoch"] + if parsed.pre is not None and parsed.post is not None: + # Such versions (a fix of a pre-release) are not needed here and are + # easy to get wrong, so they are not made. + return [f"{version} is both a pre-release and a post-release"] + return [] + + +def _is_fragment(name: str, types: list[str]) -> bool: + # towncrier names fragments . or ... + return not name.startswith(".") and any( + part in types for part in name.split(".")[1:] + ) + + +def next_development_version(version: str) -> str: + """The development version after version, as zest.releaser makes it. + + 1.0.0b1 gives 1.0.0b2.dev0, 1.0.0 gives 1.0.1.dev0, 1.0.0.post1 gives + 1.0.0.post2.dev0. + """ + if _version_errors(version): + raise ValueError(f"{version} is not a version that can be released") + parsed = Version(version) + release = ".".join(str(part) for part in parsed.release) + if parsed.pre is not None: + letter, number = parsed.pre + return f"{release}{letter}{number + 1}.dev0" + if parsed.post is not None: + return f"{release}.post{parsed.post + 1}.dev0" + parts = list(parsed.release) + parts[-1] += 1 + return ".".join(str(part) for part in parts) + ".dev0" + + +def is_prerelease(version: str) -> bool: + """Whether version is a pre-release (a, b or rc) for the GitHub Release.""" + return Version(version).is_prerelease + + +def set_version(text: str, version: str) -> str: + """Return the text of pyproject.toml with version in [project]. + + Changes only that line, so that the rest of the file stays as it is. + """ + table = _PROJECT_TABLE.search(text) + if table is None: + raise ValueError("No [project] table") + end = _NEXT_TABLE.search(text, table.end()) + stop = len(text) if end is None else end.start() + line = _VERSION_LINE.search(text, table.end(), stop) + if line is None: + raise ValueError("No version in [project]") + return f'{text[: line.start()]}{line.group(1)}"{version}"{text[line.end() :]}' + + +def fragment_types(text: str) -> list[str]: + """The fragment types (directories) of towncrier in pyproject.toml.""" + types = tomllib.loads(text)["tool"]["towncrier"]["type"] + return [str(item["directory"]) for item in types] + + +def _project_version() -> str: + data = tomllib.loads(PYPROJECT.read_text(encoding="utf-8")) + return str(data["project"]["version"]) + + +def _write_version(version: str) -> None: + PYPROJECT.write_text( + set_version(PYPROJECT.read_text(encoding="utf-8"), version), + encoding="utf-8", + ) + if _project_version() != version: + raise SystemExit(f"pyproject.toml does not have the version {version}") + + +def _remote_tags() -> list[str]: + out = subprocess.run( + ["git", "ls-remote", "--tags", "--refs", "origin"], # noqa: S607 + check=True, + capture_output=True, + text=True, + ).stdout + return [line.split("refs/tags/", 1)[1] for line in out.splitlines()] + + +def _check(version: str) -> int: + text = PYPROJECT.read_text(encoding="utf-8") + news = Path(tomllib.loads(text)["tool"]["towncrier"]["directory"]) + errors = check_release( + version, + _project_version(), + _remote_tags(), + sorted(path.name for path in news.iterdir() if path.is_file()), + fragment_types(text), + os.environ.get("GITHUB_REF", ""), + ) + for error in errors: + print(f"::error::{error}") + return 1 if errors else 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "command", + choices=["check", "set-version", "back-to-development", "prerelease"], + ) + parser.add_argument("version") + args = parser.parse_args(argv) + if args.command == "check": + return _check(args.version) + if args.command == "set-version": + _write_version(args.version) + return 0 + if args.command == "prerelease": + print("true" if is_prerelease(args.version) else "false") + return 0 + current = _project_version() + if current != args.version: + print(f"::error::pyproject.toml has {current}, not {args.version}") + return 1 + following = next_development_version(args.version) + _write_version(following) + print(following) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/plone-package.yml b/.github/workflows/plone-package.yml index 33bce81..fa62394 100644 --- a/.github/workflows/plone-package.yml +++ b/.github/workflows/plone-package.yml @@ -10,8 +10,13 @@ on: pull_request: branches: [main] workflow_dispatch: - # release.yml runs the tests before it builds. + # release.yml runs the tests of the release commit before it builds. workflow_call: + inputs: + ref: + description: The commit to test (empty for the commit of the run) + type: string + default: '' permissions: contents: read @@ -27,6 +32,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ inputs.ref }} persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -41,6 +47,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ inputs.ref }} persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -55,6 +62,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ inputs.ref }} persist-credentials: false # java/build.sh uses the pinned Maven image, not the runner's Maven # and JDK, so the jar must be the same as the committed one. @@ -103,6 +111,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ inputs.ref }} persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 45d0d29..d2b831d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,55 +1,208 @@ name: Release -# Publishes to PyPI with Trusted Publishing when zest.releaser's fullrelease -# pushes a version tag such as 2.0.0b1 (no "v" prefix). +# Makes a release from main when it is run by hand (Actions, "Release", "Run +# workflow") with the version to release, such as 2.0.0b2: +# +# 1. prepare: check the version, write the news fragments into CHANGELOG.md, +# set the version, commit "Preparing release " and tag it (no +# "v" prefix). push-release pushes both. +# 2. test, build: run the tests of that commit, build and check the sdist +# and the wheel. +# 3. publish: upload them to PyPI with Trusted Publishing, after a reviewer +# of the pypi environment approves it. +# 4. sign: sign them with Sigstore. github-release makes a GitHub Release. +# 5. development: set the next development version and commit it. +# push-development pushes it. +# +# All of it is one workflow because a tag pushed with GITHUB_TOKEN does not +# start other workflows. The steps that need code are in +# .github/scripts/release.py (tested in tests/test_release_script.py). # # Every action is pinned to a full commit SHA of a release that has been out # for at least two weeks (cooldown). The comment after each SHA is the tag. +# +# Only push-release, push-development and github-release may write to the +# repository. They run nothing but actions/checkout, +# actions/download-artifact, git and gh: the commits come from the jobs +# before them as git bundles, which they check, and the token is given to +# the step that uses it only. No checkout keeps credentials. on: - push: - tags: - - '[0-9]*' + workflow_dispatch: + inputs: + version: + description: Version to release, such as 2.0.0b2 (PEP 440, no "v") + required: true + type: string permissions: contents: read -# Never cancel a release half way. The group must differ from the one in -# plone-package.yml, which runs inside this workflow (group names ignore case). +# One release at a time, and never cancel one half way. The group must differ +# from the one in plone-package.yml, which runs inside this workflow (group +# names ignore case). concurrency: - group: publish-${{ github.ref }} + group: release cancel-in-progress: false +env: + # tomllib needs Python 3.11 or later. + RELEASE_PYTHON: '3.12' + PACKAGING: packaging==26.3 + jobs: + prepare: + runs-on: ubuntu-24.04 + outputs: + sha: ${{ steps.commit.outputs.sha }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + version: '0.13.4' + # The sha256 of uv-x86_64-unknown-linux-gnu.tar.gz of that version, in + # hex without "sha256:" (setup-uv v10.0.1 knows none for 0.13.4). All + # the jobs run on ubuntu-24.04, which is x86_64. Change it with the version. + checksum: '2b428396f12572d4429cf210ca234dd599a33b3af0958a4d5ed569d63e3f344f' + enable-cache: false + - name: Check the version + env: + VERSION: ${{ inputs.version }} + run: | + uv run --no-project --python "$RELEASE_PYTHON" --with "$PACKAGING" \ + .github/scripts/release.py check "$VERSION" + - name: Write the news fragments into CHANGELOG.md + env: + VERSION: ${{ inputs.version }} + run: uvx towncrier@26.9.0 build --version "$VERSION" --yes + - name: Set the version + env: + VERSION: ${{ inputs.version }} + run: | + uv run --no-project --python "$RELEASE_PYTHON" --with "$PACKAGING" \ + .github/scripts/release.py set-version "$VERSION" + git diff --stat + - name: Commit, tag and bundle + id: commit + env: + VERSION: ${{ inputs.version }} + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git commit --all --message "Preparing release $VERSION" + git tag --annotate "$VERSION" --message "Tagging $VERSION" + git bundle create "$RUNNER_TEMP/release.bundle" \ + "refs/tags/$VERSION" "^$GITHUB_SHA" + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-bundle + path: ${{ runner.temp }}/release.bundle + if-no-files-found: error + retention-days: 7 + + push-release: + needs: prepare + runs-on: ubuntu-24.04 + permissions: + # Push the release commit to main and the tag + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-bundle + path: ${{ runner.temp }} + - name: Take the commit and the tag from the bundle + env: + VERSION: ${{ inputs.version }} + SHA: ${{ needs.prepare.outputs.sha }} + run: | + git bundle verify "$RUNNER_TEMP/release.bundle" + git fetch "$RUNNER_TEMP/release.bundle" "refs/tags/$VERSION:refs/tags/$VERSION" + tagged=$(git rev-parse "refs/tags/$VERSION^{commit}") + # One commit with one parent, the commit of this run + if [ "$(git rev-list --parents -n 1 "$tagged")" != "$SHA $GITHUB_SHA" ]; then + echo "::error::The bundle does not have the release commit $SHA on $GITHUB_SHA" + exit 1 + fi + # It changes pyproject.toml and CHANGELOG.md and removes news + # fragments, nothing else. + other=$(git diff --no-renames --name-status "$GITHUB_SHA" "$tagged" | awk -F '\t' ' + !(($1 == "M" && ($2 == "pyproject.toml" || $2 == "CHANGELOG.md")) || + ($1 == "D" && $2 ~ /^news\/[^.\/][^\/]*$/))') + if [ -n "$other" ]; then + echo "::error::The release commit changes other files:" + echo "$other" + exit 1 + fi + - name: Push + env: + VERSION: ${{ inputs.version }} + SHA: ${{ needs.prepare.outputs.sha }} + GH_TOKEN: ${{ github.token }} + run: | + auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 --wrap=0) + echo "::add-mask::$auth" + # Both or none: if main has moved since the run started, nothing is + # pushed. + git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic $auth" \ + push --atomic origin "$SHA:refs/heads/main" "refs/tags/$VERSION" + test: + needs: [prepare, push-release] # actionlint 1.7.12 does not know the self-repository syntax ($/...) yet. # A reusable workflow given with ./ is read from the commit of this run, # not from the files of a runner. uses: ./.github/workflows/plone-package.yml # zizmor: ignore[self-repository] + with: + ref: ${{ needs.prepare.outputs.sha }} build: - needs: test - runs-on: ubuntu-latest + needs: [prepare, test] + runs-on: ubuntu-24.04 + outputs: + prerelease: ${{ steps.prerelease.outputs.prerelease }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ needs.prepare.outputs.sha }} persist-credentials: false - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: + version: '0.13.4' + # The sha256 of uv-x86_64-unknown-linux-gnu.tar.gz of that version, in + # hex without "sha256:" (setup-uv v10.0.1 knows none for 0.13.4). All + # the jobs run on ubuntu-24.04, which is x86_64. Change it with the version. + checksum: '2b428396f12572d4429cf210ca234dd599a33b3af0958a4d5ed569d63e3f344f' enable-cache: false - name: Check that the tag is the version in pyproject.toml env: - TAG: ${{ github.ref_name }} + TAG: ${{ inputs.version }} run: | version=$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])') if [ "$version" != "$TAG" ]; then echo "::error::The tag $TAG is not the version $version in pyproject.toml" exit 1 fi + - name: Tell whether it is a pre-release + id: prerelease + env: + TAG: ${{ inputs.version }} + run: | + prerelease=$(uv run --no-project --python "$RELEASE_PYTHON" --with "$PACKAGING" \ + .github/scripts/release.py prerelease "$TAG") + echo "prerelease=$prerelease" >> "$GITHUB_OUTPUT" - name: Build the sdist and the wheel run: uv build --out-dir dist - name: Check the distributions run: uvx twine@7.0.0 check --strict dist/* + - name: Check that the sdist has the files of git that it should have + run: uvx check-manifest@0.51 - name: Check that the wheel contains the jar run: | files=$(unzip -Z1 dist/*.whl) @@ -58,15 +211,36 @@ jobs: echo "::error::$name is not in the wheel" exit 1 fi + - name: Take the notes of this version from CHANGELOG.md + env: + TAG: ${{ inputs.version }} + run: | + # The section starts at "## ()" and ends before the + # next "## " heading. + mkdir notes + awk -v heading="## $TAG (" ' + index($0, heading) == 1 { found = 1; next } + found && /^## / { exit } + found { print } + ' CHANGELOG.md > notes/notes.md + if ! grep -q '[^[:space:]]' notes/notes.md; then + echo "::error::No section for $TAG in CHANGELOG.md" + exit 1 + fi - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: dist path: dist/ if-no-files-found: error + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: notes + path: notes/notes.md + if-no-files-found: error publish: needs: build - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 environment: name: pypi url: https://pypi.org/p/c2.transform.msoffice @@ -79,19 +253,19 @@ jobs: name: dist path: dist/ - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + # A re-run after a partial upload uploads only the missing files. + skip-existing: true - github-release: + sign: needs: publish - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: - # gh release create - contents: write + # actions/download-artifact needs no more + contents: read # Sigstore id-token: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: dist @@ -102,29 +276,40 @@ jobs: inputs: >- ./dist/*.tar.gz ./dist/*.whl - - name: Take the notes of this version from CHANGELOG.md - env: - TAG: ${{ github.ref_name }} - run: | - # The section starts at "## ()" and ends before the - # next "## " heading. - awk -v heading="## $TAG (" ' - index($0, heading) == 1 { found = 1; next } - found && /^## / { exit } - found { print } - ' CHANGELOG.md > "$RUNNER_TEMP/notes.md" - if ! grep -q '[^[:space:]]' "$RUNNER_TEMP/notes.md"; then - echo "::error::No section for $TAG in CHANGELOG.md" - exit 1 - fi + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: signatures + path: dist/*.sigstore.json + if-no-files-found: error + + github-release: + needs: [build, sign] + runs-on: ubuntu-24.04 + permissions: + # gh release create + contents: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dist + path: dist/ + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: signatures + path: dist/ + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: notes + path: ${{ runner.temp }} - name: Create the GitHub Release env: GH_TOKEN: ${{ github.token }} - TAG: ${{ github.ref_name }} + TAG: ${{ inputs.version }} + PRERELEASE: ${{ needs.build.outputs.prerelease }} run: | - # A version with a, b or rc (PEP 440) is a pre-release. + ls -l dist/ prerelease=() - if [[ "$TAG" =~ (a|b|rc)[0-9]+$ ]]; then + if [ "$PRERELEASE" = true ]; then prerelease=(--prerelease) fi gh release create "$TAG" dist/* \ @@ -133,3 +318,110 @@ jobs: --title "$TAG" \ --notes-file "$RUNNER_TEMP/notes.md" \ "${prerelease[@]}" + + development: + needs: [prepare, github-release] + runs-on: ubuntu-24.04 + outputs: + sha: ${{ steps.commit.outputs.sha }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.prepare.outputs.sha }} + persist-credentials: false + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + version: '0.13.4' + # The sha256 of uv-x86_64-unknown-linux-gnu.tar.gz of that version, in + # hex without "sha256:" (setup-uv v10.0.1 knows none for 0.13.4). All + # the jobs run on ubuntu-24.04, which is x86_64. Change it with the version. + checksum: '2b428396f12572d4429cf210ca234dd599a33b3af0958a4d5ed569d63e3f344f' + enable-cache: false + - name: Set the development version + id: next + env: + VERSION: ${{ inputs.version }} + run: | + next=$(uv run --no-project --python "$RELEASE_PYTHON" --with "$PACKAGING" \ + .github/scripts/release.py back-to-development "$VERSION") + echo "version=$next" >> "$GITHUB_OUTPUT" + git diff + - name: Commit and bundle + id: commit + env: + NEXT: ${{ steps.next.outputs.version }} + SHA: ${{ needs.prepare.outputs.sha }} + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git switch --create development + git commit --all --message "Back to development: ${NEXT%.dev0}" + git bundle create "$RUNNER_TEMP/development.bundle" \ + refs/heads/development "^$SHA" + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: development-bundle + path: ${{ runner.temp }}/development.bundle + if-no-files-found: error + retention-days: 7 + + push-development: + needs: [prepare, development] + runs-on: ubuntu-24.04 + permissions: + # Push the commit with the development version to main + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.prepare.outputs.sha }} + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: development-bundle + path: ${{ runner.temp }} + - name: Stop if main has moved since the release commit + env: + SHA: ${{ needs.prepare.outputs.sha }} + VERSION: ${{ inputs.version }} + run: | + main=$(git ls-remote origin refs/heads/main | cut -f1) + if [ "$main" != "$SHA" ]; then + echo "::error::main is at $main, not at the release commit $SHA. The release is done; set the development version by hand." + echo "On an up-to-date main: run '.github/scripts/release.py back-to-development $VERSION'" + echo "(it needs the packaging distribution), commit pyproject.toml with the" + echo "message 'Back to development: ' and push." + exit 1 + fi + - name: Take the commit from the bundle + env: + SHA: ${{ needs.prepare.outputs.sha }} + NEXT_SHA: ${{ needs.development.outputs.sha }} + run: | + git bundle verify "$RUNNER_TEMP/development.bundle" + git fetch "$RUNNER_TEMP/development.bundle" refs/heads/development:refs/heads/development + head=$(git rev-parse refs/heads/development) + # One commit with one parent, the release commit + if [ "$(git rev-list --parents -n 1 "$head")" != "$NEXT_SHA $SHA" ]; then + echo "::error::The bundle does not have the development commit $NEXT_SHA on $SHA" + exit 1 + fi + # It changes pyproject.toml, nothing else. + other=$(git diff --no-renames --name-status "$SHA" "$head" | awk -F '\t' ' + !($1 == "M" && $2 == "pyproject.toml")') + if [ -n "$other" ]; then + echo "::error::The development commit changes other files:" + echo "$other" + exit 1 + fi + - name: Push + env: + NEXT_SHA: ${{ needs.development.outputs.sha }} + GH_TOKEN: ${{ github.token }} + run: | + auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 --wrap=0) + echo "::add-mask::$auth" + # Not forced: if main has moved since the check, the push fails. + git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic $auth" \ + push origin "$NEXT_SHA:refs/heads/main" diff --git a/AGENTS.md b/AGENTS.md index 985dd3d..579b948 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,39 +53,98 @@ output changes, find out why before you change a file in `java/expected/`. ## Releasing -Releases are made with zest.releaser on a maintainer's machine and published -to PyPI by `.github/workflows/release.yml` with Trusted Publishing; no PyPI -token is used. The version is kept only in `pyproject.toml`. - -1. Install the tools of the `release` extra in a virtual environment that - has pip (check-manifest builds the sdist in an isolated environment made - with pip): - - ```shell - pip install "zest.releaser[recommended]" zestreleaser.towncrier zest.pocompile - ``` - - On an up-to-date `main` with a clean working tree, run `fullrelease`. - It runs check-manifest and pyroma, writes the news fragments into - `CHANGELOG.md`, sets the version, commits, makes the tag (such as - `2.0.0b1`, without `v`), sets the next development version and asks - whether to push. Answer yes: it runs `git push` and `git push --tags`. - It does not upload anything (`release = false`). -2. The tag starts the Release workflow on GitHub Actions: the tests of - `plone-package.yml`, then the build (it stops if the tag is not the - version in `pyproject.toml` or the wheel has no `msdoc2txt.jar`). -3. The `publish` job waits for a reviewer of the `pypi` environment to - approve it, then uploads the sdist and the wheel to PyPI. -4. The `github-release` job signs them with Sigstore and makes a GitHub - Release with the section of this version from `CHANGELOG.md` and the - files. Versions with `a`, `b` or `rc` are marked as pre-releases. - -Once, before the first release: - -- On PyPI, add a trusted publisher to the project: owner `cmscom`, +Releases are made only by `.github/workflows/release.yml` on GitHub +Actions; nothing is needed on a maintainer's machine. It publishes to PyPI +with Trusted Publishing, so no PyPI token is used. The version is kept only +in `pyproject.toml`. The steps that need code are in +`.github/scripts/release.py`, tested by `tests/test_release_script.py`. + +1. Make sure that `main` has the news fragments of the release in `news/`. +2. In GitHub: **Actions**, **Release**, **Run workflow**, branch `main`, + enter the version (such as `2.0.0b2`: PEP 440 in its normalized form, + without `v`) and run it. +3. `prepare` stops unless the version is after the one in `pyproject.toml`, + has no tag yet, is neither a development version nor both a pre-release + and a post-release (such as `2.0.0b2.post1`), and the run is on `main`, + with news fragments. Then it writes the fragments into `CHANGELOG.md`, + sets the version, commits "Preparing release " as + github-actions[bot] and makes the annotated tag ``. + `push-release` pushes both to GitHub. +4. `test` runs the tests of `plone-package.yml` on that commit, and + `build` builds and checks the sdist and the wheel (the wheel must + contain `msdoc2txt.jar`). +5. `publish` waits for a reviewer of the `pypi` environment. Review the + deployment and approve it; it then uploads to PyPI. +6. `sign` signs the files with Sigstore, and `github-release` makes a + GitHub Release with them, their signatures and the section of this + version from `CHANGELOG.md`. Versions with `a`, `b` or `rc` are marked + as pre-releases. +7. `development` sets the next development version (`2.0.0b2` gives + `2.0.0b3.dev0`, `2.0.0` gives `2.0.1.dev0`, `2.0.0.post1` gives + `2.0.0.post2.dev0`) and commits "Back to development: ". + `push-development` pushes it. If `main` has moved since the release + commit, it stops without pushing and says in its log how to do it by + hand. + +Only `push-release`, `push-development` and `github-release` can write to +the repository. They run only `actions/checkout`, +`actions/download-artifact`, `git` and `gh`. The push jobs get the commits +of the jobs before them as git bundles and push them only if each is one +commit on the expected parent that changes only the expected files +(`pyproject.toml`, `CHANGELOG.md` and removed news fragments for the +release, `pyproject.toml` for the development version). Keep it so: do not +add steps with other actions or tools to these jobs. + +Only one release runs at a time. A run started while another release runs +waits (GitHub keeps only the latest waiting run and cancels older ones), +then starts from the `main` of the time it was started; its `push-release` +fails because `main` has moved, and nothing is pushed. Run the workflow +again. + +The `setup-uv` steps pin uv with `version` and `checksum`. When you change +the version, change the checksum with it. + +### When a release fails or is rejected + +- Before `push-release` pushed, nothing has changed: fix the cause and run + the workflow again. +- After `push-release` pushed, but before `publish` ran (the tests or the + build failed, or the deployment was rejected): PyPI has nothing of this + version. Delete the tag (`git push origin :refs/tags/` and + `git tag -d `), revert "Preparing release " on `main` + (this brings the news fragments back) and push, fix the cause, and run + the workflow again with the same version. +- If `publish` ran, look at for + the files of this version. If even one of them is there, do not delete + the tag: PyPI never accepts the same file again. Re-run `publish` to + upload the missing files (it skips the existing ones), then re-run the + jobs after it. A re-run uses the files of the same run: before you go + on, check that the sha256 of each file that PyPI has already is the one + of the file in the `dist` artifact of the run. To fix the release + itself, release the next version. +- If a re-run of `github-release` fails because the release already + exists, add the missing files to it with + `gh release upload --clobber` (the files of the + `dist` artifact and their `.sigstore.json` of the `signatures` artifact + of the run), or delete the GitHub Release and re-run the job. +- If `push-development` failed, re-run it, or set the development version + by hand as its log says. + +### Settings needed once + +- PyPI: the trusted publisher of `c2.transform.msoffice` is owner `cmscom`, repository `c2.transform.msoffice`, workflow `release.yml`, environment `pypi`. -- On GitHub, add the environment `pypi` with required reviewers. +- GitHub: the environment `pypi` with required reviewers. The workflow runs + on the branch `main`, so a deployment rule of the environment must allow + `main` (not only tags). +- `push-release` and `push-development` push to `main` with `GITHUB_TOKEN`. + If `main` gets a branch protection rule or a ruleset, it must let these + pushes through (as an exception for GitHub Actions); otherwise the + release stops at the push. Check this when you add the protection. + Commits pushed with `GITHUB_TOKEN` do not start other workflows, so + `plone-package.yml` does not run again for them; the release runs those + tests itself. ## Test samples diff --git a/pyproject.toml b/pyproject.toml index 2d5c181..83bd985 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,17 +46,15 @@ dependencies = [ [project.optional-dependencies] test = [ "horse-with-no-namespace", + # packaging and tomli: .github/scripts/release.py, which a test imports + "packaging", "plone.api", "plone.app.contenttypes", "plone.app.testing", "pytest", "pytest-cov", "pytest-plone>=0.5.0", -] -release = [ - "zest.releaser[recommended]", - "zestreleaser.towncrier", - "zest.pocompile", + "tomli; python_version < '3.11'", ] [dependency-groups] @@ -213,6 +211,8 @@ module = [ "Products.PortalTransforms.interfaces", "Products.PortalTransforms.TransformEngine", "pytest_plone", + # Only on Python 3.10, for .github/scripts/release.py + "tomli", "zope.component", "zope.configuration.config", "zope.interface", @@ -237,7 +237,3 @@ ignore = [ "java/**", "news/**", ] - -[tool.zest-releaser] -# fullrelease makes the tag; the release workflow uploads to PyPI. -release = false diff --git a/tests/test_release_script.py b/tests/test_release_script.py new file mode 100644 index 0000000..87118ce --- /dev/null +++ b/tests/test_release_script.py @@ -0,0 +1,226 @@ +"""Tests of .github/scripts/release.py, which the release workflow runs.""" + +from pathlib import Path +from types import ModuleType + +import importlib.util +import pytest +import sys + + +SCRIPT = Path(__file__).parents[1] / ".github" / "scripts" / "release.py" + + +def load_script() -> ModuleType: + spec = importlib.util.spec_from_file_location("release_script", SCRIPT) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["release_script"] = module + spec.loader.exec_module(module) + return module + + +release = load_script() + +TYPES = ["breaking", "feature", "bugfix", "internal", "documentation", "tests"] +MAIN = "refs/heads/main" + + +def check( + version: str = "1.0.0b2", + current: str = "1.0.0b2.dev0", + tags: tuple[str, ...] = ("1.0.0b1",), + names: tuple[str, ...] = ("+fix.bugfix",), + ref: str = MAIN, +) -> list[str]: + errors: list[str] = release.check_release( + version, current, list(tags), list(names), TYPES, ref + ) + return errors + + +def test_a_good_release_has_no_errors(): + assert check() == [] + + +@pytest.mark.parametrize( + ("version", "current"), + [ + ("1.0.0b2", "1.0.0b2.dev0"), + ("1.0.0", "1.0.0b2.dev0"), + ("1.0.0rc1", "1.0.0b2.dev0"), + ("1.1.0", "1.0.0b2.dev0"), + ("1.0.1", "1.0.0"), + ("1.0.0.post1", "1.0.0"), + ("2.0", "1.9.9"), + ], +) +def test_versions_after_the_current_one(version, current): + assert check(version, current) == [] + + +@pytest.mark.parametrize( + ("version", "error"), + [ + ("", "'' is not a PEP 440 version"), + ("v1.0.0b2", "'v1.0.0b2' is not in the normalized form '1.0.0b2'"), + ("1.0.0-b2", "'1.0.0-b2' is not in the normalized form '1.0.0b2'"), + ("1.0.0B2", "'1.0.0B2' is not in the normalized form '1.0.0b2'"), + (" 1.0.0b2", "' 1.0.0b2' is not in the normalized form '1.0.0b2'"), + ("1.0.0beta2", "'1.0.0beta2' is not in the normalized form '1.0.0b2'"), + ("1.0.0b2\n", "'1.0.0b2\\n' is not in the normalized form '1.0.0b2'"), + ("1.0.0 b2", "'1.0.0 b2' is not a PEP 440 version"), + ("1.0.0b2;x", "'1.0.0b2;x' is not a PEP 440 version"), + ("latest", "'latest' is not a PEP 440 version"), + ("1.0.0b3.dev0", "1.0.0b3.dev0 is a development version"), + ("1.0.0b2+local", "1.0.0b2+local has a local part"), + ("1!1.0.0", "1!1.0.0 has an epoch"), + ("1.0.0b2.post1", "1.0.0b2.post1 is both a pre-release and a post-release"), + ("1.0.0rc1.post2", "1.0.0rc1.post2 is both a pre-release and a post-release"), + ], +) +def test_bad_versions(version, error): + assert check(version) == [error] + + +@pytest.mark.parametrize( + ("version", "current"), + [ + ("1.0.0b1", "1.0.0b2.dev0"), + ("1.0.0b2", "1.0.0b2"), + ("1.0.0a9", "1.0.0b2.dev0"), + ("0.9", "1.0.0b2.dev0"), + ], +) +def test_versions_not_after_the_current_one(version, current): + assert check(version, current, tags=()) == [ + f"{version} is not after the version {current} in pyproject.toml" + ] + + +def test_an_existing_tag(): + assert check(tags=("1.0.0b1", "1.0.0b2")) == ["The tag 1.0.0b2 exists already"] + + +@pytest.mark.parametrize( + "names", + [ + (), + (".gitkeep", ".changelog_template.jinja"), + ("README.md", "+fix.unknown", "feature", ".hidden.feature"), + ], +) +def test_no_news_fragments(names): + assert check(names=names) == ["There are no news fragments in news/"] + + +@pytest.mark.parametrize( + "name", + ["+fix.bugfix", "12.feature", "+x.breaking", "12.feature.1", "+a.b.documentation"], +) +def test_news_fragments(name): + assert check(names=(".gitkeep", name)) == [] + + +@pytest.mark.parametrize("ref", ["refs/heads/release", "refs/tags/1.0.0b1", ""]) +def test_only_from_main(ref): + assert check(ref=ref) == [f"Run the release from main, not from {ref!r}"] + + +def test_all_errors_at_once(): + assert check( + version="1.0.0b1", tags=("1.0.0b1",), names=(), ref="refs/heads/x" + ) == [ + "Run the release from main, not from 'refs/heads/x'", + "1.0.0b1 is not after the version 1.0.0b2.dev0 in pyproject.toml", + "The tag 1.0.0b1 exists already", + "There are no news fragments in news/", + ] + + +@pytest.mark.parametrize( + ("version", "expected"), + [ + ("1.0.0b1", "1.0.0b2.dev0"), + ("1.0.0a3", "1.0.0a4.dev0"), + ("1.0.0rc1", "1.0.0rc2.dev0"), + ("1.0.0", "1.0.1.dev0"), + ("1.0", "1.1.dev0"), + ("2", "3.dev0"), + ("1.0.9", "1.0.10.dev0"), + ("1.0.0.post1", "1.0.0.post2.dev0"), + ("1.0.0b9", "1.0.0b10.dev0"), + ], +) +def test_next_development_version(version, expected): + assert release.next_development_version(version) == expected + + +def test_next_development_version_after_a_post_release(): + # A post-release fixes a release, so the next one is another + # post-release, not the next release. + assert release.next_development_version("1.0.0.post1") == "1.0.0.post2.dev0" + + +@pytest.mark.parametrize( + "version", ["1.0.0b2.post1", "1.0.0b3.dev0", "1.0.0+local", "1!1.0.0"] +) +def test_no_next_development_version(version): + with pytest.raises(ValueError, match="is not a version that can be released"): + release.next_development_version(version) + + +@pytest.mark.parametrize( + ("version", "expected"), + [ + ("1.0.0a1", True), + ("1.0.0b2", True), + ("1.0.0rc1", True), + ("1.0.0", False), + ("1.0.0.post1", False), + ("2", False), + ], +) +def test_is_prerelease(version, expected): + assert release.is_prerelease(version) is expected + + +PYPROJECT = """\ +[build-system] +requires = ["setuptools>=69.0"] + +[project] +name = "c2.transform.msoffice" +version = "1.0.0b2.dev0" +description = "version = \\"not this\\"" + +[tool.other] +version = "9.9" +""" + + +def test_set_version_changes_only_the_project_version(): + assert release.set_version(PYPROJECT, "1.0.0b2") == PYPROJECT.replace( + 'version = "1.0.0b2.dev0"', 'version = "1.0.0b2"' + ) + + +def test_set_version_without_a_project_version(): + text = '[project]\nname = "x"\n\n[tool.other]\nversion = "9.9"\n' + with pytest.raises(ValueError, match=r"No version in \[project\]"): + release.set_version(text, "1.0.0") + + +def test_fragment_types_from_pyproject(): + text = ( + '[tool.towncrier]\ndirectory = "news/"\n\n' + '[[tool.towncrier.type]]\ndirectory = "feature"\nname = "New"\n\n' + '[[tool.towncrier.type]]\ndirectory = "bugfix"\nname = "Fixes"\n' + ) + assert release.fragment_types(text) == ["feature", "bugfix"] + + +def test_fragment_types_of_this_package(): + text = (Path(__file__).parents[1] / "pyproject.toml").read_text(encoding="utf-8") + assert release.fragment_types(text) == TYPES diff --git a/tox.ini b/tox.ini index 67d7aac..858c299 100644 --- a/tox.ini +++ b/tox.ini @@ -48,8 +48,8 @@ deps = ruff commands = - ruff check --no-fix src tests - ruff format --check --diff src tests + ruff check --no-fix src tests .github/scripts + ruff format --check --diff src tests .github/scripts [testenv:mypy] @@ -60,7 +60,7 @@ setenv = deps = -rrequirements_plone62.txt commands = - mypy src tests + mypy src tests .github/scripts [testenv:ruff-fix] @@ -72,8 +72,8 @@ deps = ruff commands = - ruff check --fix src tests - ruff format src tests + ruff check --fix src tests .github/scripts + ruff format src tests .github/scripts [testenv:coverage-report]