From d5304bf0ed80d71ac885ecf612561c49e4f62d7e Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 18 Aug 2026 13:37:02 +0900 Subject: [PATCH 1/5] Add PyPI wheel publishing flow --- .github/workflows/publish-pypi.yaml | 61 +++++++++++ make/buf/all.mk | 1 + make/buf/scripts/release.bash | 4 + packaging/python/.gitignore | 3 + packaging/python/LICENSE | 1 + packaging/python/README.md | 1 + packaging/python/buf_bin/__init__.py | 2 + packaging/python/out/bin/.gitkeep | 0 packaging/python/pyproject.toml | 44 ++++++++ packaging/python/scripts/generate_wheels.py | 112 ++++++++++++++++++++ packaging/python/uv.lock | 39 +++++++ 11 files changed, 268 insertions(+) create mode 100644 .github/workflows/publish-pypi.yaml create mode 100644 packaging/python/.gitignore create mode 120000 packaging/python/LICENSE create mode 120000 packaging/python/README.md create mode 100644 packaging/python/buf_bin/__init__.py create mode 100644 packaging/python/out/bin/.gitkeep create mode 100644 packaging/python/pyproject.toml create mode 100644 packaging/python/scripts/generate_wheels.py create mode 100644 packaging/python/uv.lock diff --git a/.github/workflows/publish-pypi.yaml b/.github/workflows/publish-pypi.yaml new file mode 100644 index 0000000000..ce71b805a0 --- /dev/null +++ b/.github/workflows/publish-pypi.yaml @@ -0,0 +1,61 @@ +# NOTE: This filename is hardcoded in our trusted publishing setup on pypi.org and test.pypi.org; +# do not change this filename without updating it in both places as well. +name: Publish to PyPI +on: + workflow_dispatch: + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - run: uv sync --frozen + + - run: uv run python scripts/generate_wheels.py + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wheels + path: dist/ + + test-publish: + needs: build + runs-on: ubuntu-latest + environment: pypi-test + permissions: + id-token: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: wheels + path: dist/ + + - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + + publish: + needs: test-publish + runs-on: ubuntu-latest + environment: pypi-release + permissions: + id-token: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: wheels + path: dist/ + + - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 + with: + skip-existing: true diff --git a/make/buf/all.mk b/make/buf/all.mk index e57ad67120..d3951fd83f 100644 --- a/make/buf/all.mk +++ b/make/buf/all.mk @@ -199,3 +199,4 @@ postupgrade:: checkandupdateprecommithooks updatebufversion: $(SED_I) -E "s/BUF_VERSION \?=.*/BUF_VERSION ?= v${RELEASE_BUF_VERSION}/" "make/go/dep_buf.mk" $(SED_I) -E "s/\# https\:\/\/github.com\/bufbuild\/buf\/releases.*/\# https\:\/\/github.com\/bufbuild\/buf\/releases $(shell date "+%Y%m%d") checked $(shell date "+%Y%m%d")/" "make/go/dep_buf.mk" + cd packaging/python && uv version ${RELEASE_BUF_VERSION} diff --git a/make/buf/scripts/release.bash b/make/buf/scripts/release.bash index 37704a7ec9..386817b383 100644 --- a/make/buf/scripts/release.bash +++ b/make/buf/scripts/release.bash @@ -184,4 +184,8 @@ for file in $(find . -maxdepth 1 -type f | sed 's/^\.\///' | sort | uniq); do mv "${file}" "assets/${file}" done +pushd "${DIR}"/packaging/python >/dev/null +uv run python scripts/generate_wheels.py +popd > /dev/null + echo Upload all the files in this directory to GitHub: open "${RELEASE_DIR}/assets" diff --git a/packaging/python/.gitignore b/packaging/python/.gitignore new file mode 100644 index 0000000000..e69c7fe942 --- /dev/null +++ b/packaging/python/.gitignore @@ -0,0 +1,3 @@ +.venv +.ruff_cache +dist diff --git a/packaging/python/LICENSE b/packaging/python/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/packaging/python/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/packaging/python/README.md b/packaging/python/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/packaging/python/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/packaging/python/buf_bin/__init__.py b/packaging/python/buf_bin/__init__.py new file mode 100644 index 0000000000..b5d5057edc --- /dev/null +++ b/packaging/python/buf_bin/__init__.py @@ -0,0 +1,2 @@ +# buf-bin: Buf CLI distributed via PyPI. +# The buf binary is installed directly as a script on PATH. diff --git a/packaging/python/out/bin/.gitkeep b/packaging/python/out/bin/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packaging/python/pyproject.toml b/packaging/python/pyproject.toml new file mode 100644 index 0000000000..2c242c730b --- /dev/null +++ b/packaging/python/pyproject.toml @@ -0,0 +1,44 @@ +[project] +name = "buf-bin" +version = "1.71.0" +description = "Buf CLI distributed via PyPI" +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" +license-files = ["LICENSE"] +keywords = [ + "buf", + "bufbuild", + "protobuf", + "protoc", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Build Tools", + "Topic :: Software Development :: Code Generators", + "Topic :: Software Development :: Compilers", +] + +[project.urls] +Changelog = "https://github.com/bufbuild/buf/blob/main/CHANGELOG.md" +Documentation = "https://buf.build/docs/cli/" +Homepage = "https://buf.build/docs/cli/" +Issue = "https://github.com/bufbuild/buf/issues" +Repository = "https://github.com/bufbuild/buf" + +[dependency-groups] +dev = ["wheel"] + +[build-system] +requires = ["uv_build>=0.11.0,<0.12.0"] +build-backend = "uv_build" + +[tool.uv.build-backend] +module-name = "buf_bin" +module-root = "" +data = { scripts = "out/bin" } diff --git a/packaging/python/scripts/generate_wheels.py b/packaging/python/scripts/generate_wheels.py new file mode 100644 index 0000000000..449e66ea68 --- /dev/null +++ b/packaging/python/scripts/generate_wheels.py @@ -0,0 +1,112 @@ +import hashlib +import shutil +import subprocess +import sys +import urllib.request +from importlib.metadata import version as pkg_version +from pathlib import Path + +GITHUB_RELEASES_BASE = "https://github.com/bufbuild/buf/releases/download" + +# Maps buf platform suffix to Python wheel platform tag. +# buf binary names on GitHub releases follow the pattern: +# buf-{PLATFORM} or buf-{PLATFORM}.exe +# Check https://go.dev/wiki/MinimumRequirements#operating-systems for +# minimum OS versions, especially macOS. +PLATFORMS = [ + ("Darwin-arm64", "macosx_11_0_arm64"), + ("Darwin-x86_64", "macosx_11_0_x86_64"), + ("Linux-aarch64", "manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64"), + ("Linux-armv7", "manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l"), + ("Linux-ppc64le", "manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le"), + ("Linux-riscv64", "manylinux_2_17_riscv64.musllinux_1_2_riscv64"), + ("Linux-s390x", "manylinux_2_17_s390x.manylinux2014_s390x.musllinux_1_1_s390x"), + ("Linux-x86_64", "manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64"), + ("Windows-arm64", "win_arm64"), + ("Windows-x86_64", "win_amd64"), +] + + +def fetch_checksums(version: str) -> dict[str, str]: + url = f"{GITHUB_RELEASES_BASE}/v{version}/sha256.txt" + with urllib.request.urlopen(url) as response: + content = response.read().decode() + result = {} + for line in content.splitlines(): + if " " in line: + sha256, filename = line.split(" ", 1) + result[filename] = sha256 + return result + + +def verify_checksum(path: Path, expected: str) -> None: + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected: + msg = f"checksum mismatch for {path.name}: expected {expected}, got {actual}" + raise ValueError(msg) + + +def download(url: str, dest: Path) -> None: + with urllib.request.urlopen(url) as response, dest.open("wb") as f: + shutil.copyfileobj(response, f) + + +def main() -> None: + base_dir = Path(__file__).parent.parent + + version = pkg_version("buf-bin") + + print(f"Generating wheels for buf v{version}") + checksums = fetch_checksums(version) + + bin_dir = base_dir / "out" / "bin" + + for buf_platform, wheel_platform in PLATFORMS: + print(f"\nBuilding wheel for {buf_platform} ({wheel_platform})") + + shutil.rmtree(bin_dir, ignore_errors=True) + bin_dir.mkdir(parents=True) + + try: + ext = ".exe" if buf_platform.startswith("Windows") else "" + filename = f"buf-{buf_platform}{ext}" + dest = bin_dir / f"buf{ext}" + print(f" Downloading {GITHUB_RELEASES_BASE}/v{version}/{filename}") + download(f"{GITHUB_RELEASES_BASE}/v{version}/{filename}", dest) + verify_checksum(dest, checksums[filename]) + if not ext: + dest.chmod(0o755) + + subprocess.run( + ["uv", "build", "--wheel"], + check=True, + cwd=base_dir, + ) + + dist_dir = base_dir / "dist" + built_wheel = next(dist_dir.glob("*-py3-none-any.whl")) + + subprocess.run( + [ + sys.executable, + "-m", + "wheel", + "tags", + "--remove", + "--platform-tag", + wheel_platform, + str(built_wheel), + ], + check=True, + ) + finally: + shutil.rmtree(bin_dir, ignore_errors=True) + + bin_dir.mkdir(parents=True) + (bin_dir / ".gitkeep").touch() + + print("\nDone. Wheels written to dist/") + + +if __name__ == "__main__": + main() diff --git a/packaging/python/uv.lock b/packaging/python/uv.lock new file mode 100644 index 0000000000..6bf8bee9f7 --- /dev/null +++ b/packaging/python/uv.lock @@ -0,0 +1,39 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" + +[[package]] +name = "buf-bin" +version = "1.71.0" +source = { editable = "." } + +[package.dev-dependencies] +dev = [ + { name = "wheel" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [{ name = "wheel" }] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "wheel" +version = "0.48.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/20/50ed6bdf27dec98b568a8ae25dc599f35baa3d9709f9e83fd1edb56b9a90/wheel-0.48.0.tar.gz", hash = "sha256:94800765601e9171bf5d58d066e640662842bcedcbab982b2c90787a2c987322", size = 66471, upload-time = "2026-08-11T22:02:27.327Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/29/69cfbb602cd91690c55d38ba9fe53e6a7e76a6fa647bf38f19c138d25449/wheel-0.48.0-py3-none-any.whl", hash = "sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab", size = 33320, upload-time = "2026-08-11T22:02:26.1Z" }, +] From a5a0082ffb611cf1f0774eaaac4aa4905575c671 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Wed, 26 Aug 2026 10:33:01 +0900 Subject: [PATCH 2/5] Remove uv requirement from publishing --- .github/workflows/publish-pypi.yaml | 19 +++++++------- make/buf/all.mk | 2 +- packaging/python/.gitignore | 1 + packaging/python/uv.lock | 39 ----------------------------- 4 files changed, 12 insertions(+), 49 deletions(-) delete mode 100644 packaging/python/uv.lock diff --git a/.github/workflows/publish-pypi.yaml b/.github/workflows/publish-pypi.yaml index ce71b805a0..3dc7e92343 100644 --- a/.github/workflows/publish-pypi.yaml +++ b/.github/workflows/publish-pypi.yaml @@ -12,50 +12,51 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@v7 with: persist-credentials: false - - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - - run: uv sync --frozen + - run: uv sync - run: uv run python scripts/generate_wheels.py - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - uses: actions/upload-artifact@v7 with: name: wheels path: dist/ test-publish: + if: github.event_name == 'workflow_dispatch' needs: build runs-on: ubuntu-latest environment: pypi-test permissions: id-token: write steps: - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: actions/download-artifact@v8 with: name: wheels path: dist/ - - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 + - uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ skip-existing: true publish: - needs: test-publish + if: github.event_name != 'workflow_dispatch' runs-on: ubuntu-latest environment: pypi-release permissions: id-token: write steps: - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: actions/download-artifact@v8 with: name: wheels path: dist/ - - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 + - uses: pypa/gh-action-pypi-publish@release/v1 with: skip-existing: true diff --git a/make/buf/all.mk b/make/buf/all.mk index d3951fd83f..397339a5e1 100644 --- a/make/buf/all.mk +++ b/make/buf/all.mk @@ -199,4 +199,4 @@ postupgrade:: checkandupdateprecommithooks updatebufversion: $(SED_I) -E "s/BUF_VERSION \?=.*/BUF_VERSION ?= v${RELEASE_BUF_VERSION}/" "make/go/dep_buf.mk" $(SED_I) -E "s/\# https\:\/\/github.com\/bufbuild\/buf\/releases.*/\# https\:\/\/github.com\/bufbuild\/buf\/releases $(shell date "+%Y%m%d") checked $(shell date "+%Y%m%d")/" "make/go/dep_buf.mk" - cd packaging/python && uv version ${RELEASE_BUF_VERSION} + $(SED_I) -E "s/^version = \".*\"/version = \"${RELEASE_BUF_VERSION}\"/" "packaging/python/pyproject.toml" diff --git a/packaging/python/.gitignore b/packaging/python/.gitignore index e69c7fe942..f1c6f33013 100644 --- a/packaging/python/.gitignore +++ b/packaging/python/.gitignore @@ -1,3 +1,4 @@ .venv .ruff_cache dist +uv.lock diff --git a/packaging/python/uv.lock b/packaging/python/uv.lock deleted file mode 100644 index 6bf8bee9f7..0000000000 --- a/packaging/python/uv.lock +++ /dev/null @@ -1,39 +0,0 @@ -version = 1 -revision = 3 -requires-python = ">=3.10" - -[[package]] -name = "buf-bin" -version = "1.71.0" -source = { editable = "." } - -[package.dev-dependencies] -dev = [ - { name = "wheel" }, -] - -[package.metadata] - -[package.metadata.requires-dev] -dev = [{ name = "wheel" }] - -[[package]] -name = "packaging" -version = "26.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, -] - -[[package]] -name = "wheel" -version = "0.48.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d0/20/50ed6bdf27dec98b568a8ae25dc599f35baa3d9709f9e83fd1edb56b9a90/wheel-0.48.0.tar.gz", hash = "sha256:94800765601e9171bf5d58d066e640662842bcedcbab982b2c90787a2c987322", size = 66471, upload-time = "2026-08-11T22:02:27.327Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/29/69cfbb602cd91690c55d38ba9fe53e6a7e76a6fa647bf38f19c138d25449/wheel-0.48.0-py3-none-any.whl", hash = "sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab", size = 33320, upload-time = "2026-08-11T22:02:26.1Z" }, -] From daf25abf8ca2fdfb8a81c97785fdef998ced81c5 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Wed, 26 Aug 2026 10:34:15 +0900 Subject: [PATCH 3/5] Revert unused --- make/buf/scripts/release.bash | 4 ---- 1 file changed, 4 deletions(-) diff --git a/make/buf/scripts/release.bash b/make/buf/scripts/release.bash index 386817b383..37704a7ec9 100644 --- a/make/buf/scripts/release.bash +++ b/make/buf/scripts/release.bash @@ -184,8 +184,4 @@ for file in $(find . -maxdepth 1 -type f | sed 's/^\.\///' | sort | uniq); do mv "${file}" "assets/${file}" done -pushd "${DIR}"/packaging/python >/dev/null -uv run python scripts/generate_wheels.py -popd > /dev/null - echo Upload all the files in this directory to GitHub: open "${RELEASE_DIR}/assets" From c37895c9e54878a5d1f7a1075f62529b19c969cf Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Wed, 26 Aug 2026 10:35:04 +0900 Subject: [PATCH 4/5] update version --- packaging/python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/python/pyproject.toml b/packaging/python/pyproject.toml index 2c242c730b..a35668d6b5 100644 --- a/packaging/python/pyproject.toml +++ b/packaging/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "buf-bin" -version = "1.71.0" +version = "1.72.0" description = "Buf CLI distributed via PyPI" readme = "README.md" requires-python = ">=3.10" From 0a2988204e5d0c4f348a5787389a11baa2ed25b3 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Wed, 26 Aug 2026 10:42:06 +0900 Subject: [PATCH 5/5] license header --- packaging/python/buf_bin/__init__.py | 14 ++++++++++++++ packaging/python/scripts/generate_wheels.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/packaging/python/buf_bin/__init__.py b/packaging/python/buf_bin/__init__.py index b5d5057edc..592c0de6e4 100644 --- a/packaging/python/buf_bin/__init__.py +++ b/packaging/python/buf_bin/__init__.py @@ -1,2 +1,16 @@ +# Copyright 2020-2026 Buf Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # buf-bin: Buf CLI distributed via PyPI. # The buf binary is installed directly as a script on PATH. diff --git a/packaging/python/scripts/generate_wheels.py b/packaging/python/scripts/generate_wheels.py index 449e66ea68..e98e2f0daa 100644 --- a/packaging/python/scripts/generate_wheels.py +++ b/packaging/python/scripts/generate_wheels.py @@ -1,3 +1,17 @@ +# Copyright 2020-2026 Buf Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import hashlib import shutil import subprocess