diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..da322a9 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,10 @@ +{ + "permissions": { + "allow": [ + "Bash(.venv/bin/portable-python:*)", + "Bash(.venv/bin/pytest:*)", + "Bash(.venv/bin/python scripts/check_okf.py:*)", + "Bash(tox:*)" + ] + } +} diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md deleted file mode 100644 index ab3cfb4..0000000 --- a/ARCHITECTURE.md +++ /dev/null @@ -1,74 +0,0 @@ -## Architecture - -### Core Classes Hierarchy - -``` -ModuleBuilder (abstract base) -├── PythonBuilder (abstract, extends ModuleBuilder) -│ └── Cpython (concrete implementation in cpython.py) -└── External modules (in external/xcpython.py, xtkinter.py) - ├── Bdb, Bzip2, Gdbm, LibFFI, Mpdec, Openssl, Readline - ├── Sqlite, Uuid, Xz, Zlib, Zstd, TkInter - -BuildSetup (coordinates overall compilation) -├── BuildContext (handles isolation hacks on macOS) -├── Folders (manages build directory structure) -└── ModuleCollection (manages module selection and dependencies) - -PPG (Global state singleton) -├── config: Config (YAML configuration) -├── target: PlatformId (target OS/arch) -├── cpython: CPythonFamily (available versions) -└── families: dict (extensible family support) - -Config -├── Folders (build/dist/sources/logs directories) -├── target platform detection -├── YAML config merging (platform-specific overrides) - -PythonInspector -├── Inspect Python installation for portability -├── LibAutoCorrect (rewrite paths to be relative) -└── Report system and shared lib detection - -Tracker/Trackable -├── Categorizes found issues/objects by type -└── Provides detailed reports -``` - -### Key Design Patterns - -1. **Hierarchical Module Building**: External modules are compiled first, then Python itself, using the same build framework. - -2. **Environment Variable Injection**: `xenv_*` methods dynamically provide environment variables (CPATH, LDFLAGS, PATH, etc.) for compilation. - -3. **Platform Abstraction**: `PPG.target` (PlatformId) encapsulates platform logic. Compile methods named `_do__compile()` dispatch to platform-specific implementations. - -4. **Configuration Precedence**: YAML config supports platform-specific overrides (windows.ext, macos.env, etc.). Most specific setting wins. - -5. **Folder Masking (macOS)**: On macOS, `/usr/local` is temporarily masked with a RAM disk to prevent accidental dynamic linking. - -6. **Build Isolation**: All external modules compiled to a shared `build/deps/` folder, Python finds them via CPATH/LDFLAGS. - -7. **Lazy Version Fetching**: `VersionFamily` caches available versions, fetching from python.org on first access. - -8. **Telltale Detection**: Modules check for marker files (`m_telltale`) to determine if they're already available on the system (as shared libs). - -9. **Log Aggregation**: Each module logs to a separate file (`01-openssl.log`, `02-cpython.log`, etc.) under `build/logs/`. - - -## CI/CD - -### GitHub Actions - -**tests.yml** (main branch & PRs): -- Matrix test on py3.10, 3.11, 3.12, 3.13, 3.14 -- Runs: `uvx --with tox-uv tox -e py` -- Coverage upload to coveralls.io (parallel, then finish) -- Linter job: docs + style checks on 3.14 - -**release.yml** (version tags v*): -- Triggers on `v[0-9]*` tags -- Runs all tests + docs + style -- Builds distribution with `uv build` -- Publishes to PyPI via trusted publishing (OIDC) diff --git a/CLAUDE.md b/CLAUDE.md index 7e54760..f57975e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,76 +1,24 @@ # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +Guidance for Claude Code working in this repository. -## What This Project Does +`portable-python` is a CLI and Python library that compiles portable (statically-linked, relocatable) CPython binaries from source — see [`docs/overview.md`](docs/overview.md) for the full picture. -portable-python is a CLI tool and Python library for compiling portable CPython binaries from source. Binaries are statically linked so they can be extracted to any folder and used without installation. Supports Linux and macOS (not Windows). +## Documentation lives in `docs/` -## Common Commands +The authoritative, self-contained documentation is the OKF knowledge bundle under **`docs/`** — start at [`docs/index.md`](docs/index.md). **Don't duplicate it in this file — link to it.** When you change behavior, update the matching `docs/` entry and add a line to [`docs/log.md`](docs/log.md). -```bash -# Run all tests (tox manages envs for py310-314, coverage, docs, style) -tox +| Looking for… | Go to | +|--------------|-------| +| What it does, guiding principles, end-to-end build flow | [`docs/overview.md`](docs/overview.md) | +| Core classes & how they collaborate | [`docs/architecture/`](docs/architecture/index.md) | +| Concepts: portability, static linking, telltale detection, folder masking, ppp-marker, build layout | [`docs/concepts/`](docs/concepts/index.md) | +| CLI commands & options | [`docs/cli/`](docs/cli/index.md) | +| External modules (the statically-linked C libraries) | [`docs/modules/`](docs/modules/index.md) | +| Configuration (`portable-python.yml`) | [`docs/configuration/`](docs/configuration/index.md) | +| Dev setup, tests, code style, debugging, Docker, CI/CD, and common tasks | [`docs/guides/`](docs/guides/index.md) | -# Run tests for a single Python version -tox -e py313 +## Working in this repo -# Run a single test -pytest tests/test_build.py::test_build_rc -vv - -# Lint check / auto-fix -tox -e style -tox -e reformat - -# Run tests directly (from venv) -pytest tests/ - -# CI uses: uvx --with tox-uv tox -e py -``` - -## Architecture - -**Key classes:** - -- `PPG` (versions.py) — Global singleton holding config, target platform, and version families. All modules access shared state through this. -- `BuildSetup` (__init__.py) — Coordinates overall compilation: downloads sources, builds external modules, then CPython. -- `ModuleBuilder` (__init__.py) — Abstract base for anything that gets compiled. Both external C modules and Python itself extend this. -- `PythonBuilder` (__init__.py) — Extends ModuleBuilder for Python implementations. -- `Cpython` (cpython.py) — Concrete PythonBuilder that handles CPython's configure/make/install, optimization, and finalization. -- `Config` (config.py) — Loads and merges YAML configuration (portable-python.yml) with platform-specific overrides. -- `PythonInspector` (inspector.py) — Validates portability of a built Python by checking shared library dependencies and paths. - -**External modules** (external/xcpython.py): `Bdb`, `Bzip2`, `Gdbm`, `LibFFI`, `Mpdec`, `Openssl`, `Readline`, `Sqlite`, `Uuid`, `Xz`, `Zlib`, `Zstd` — each is a ModuleBuilder subclass that compiles a C library statically before CPython is built. - -**Key patterns:** - -- Platform-specific compile logic uses `_do_linux_compile()` / `_do_macos_compile()` method dispatch. -- Environment injection: `xenv_*` methods provide CPATH, LDFLAGS, PATH etc. for compilation. -- On macOS, `/usr/local` is masked with a RAM disk (`FolderMask`) to prevent accidental dynamic linking. -- External modules compile to a shared `build/deps/` prefix; CPython finds them via CPATH/LDFLAGS. -- Telltale detection: modules check for marker files (`m_telltale`) to determine if system already has the library. -- No patches to upstream CPython source — relies solely on configure flags. - -**runez** is the foundational utility library (file ops, system info, CLI decorators, logging, Version/PythonSpec types). Check runez before reimplementing anything. - -**Additional pointers:** - -- `ModuleCollection.selected` contains only the modules chosen for a build — not all candidates. -- Build logs go to `build/logs/NN-modulename.log` (e.g. `01-openssl.log`, `02-cpython.log`). -- YAML config supports platform-specific overrides and path templates — see CONFIGURATION.md. -- See ARCHITECTURE.md for class hierarchy and design patterns, DEVELOP.md for common tasks and dependencies. - -## Testing - -- pytest with 100% code coverage target -- Tests mock `runez.run()` to avoid actual compilation — uses `--dryrun` mode -- `conftest.py` provides a `cli` fixture (from runez) and forbids HTTP calls (`GlobalHttpCalls.forbid()`) -- Sample YAML configs in `tests/sample-config*.yml` for testing configuration parsing - -## Linting - -Ruff handles all linting and formatting. Key settings in pyproject.toml: -- Line length: 140 -- McCabe complexity: max 18 -- Security checks (S rules) disabled in tests -- Numpy-style docstrings +- Check **runez** before reimplementing file / system / CLI / logging / version helpers — see [`docs/guides/local-development.md`](docs/guides/local-development.md). +- The docs use the OKF format; keep the bundle conformant (`scripts/check_okf.py docs`) when you edit it. diff --git a/CONFIGURATION.md b/CONFIGURATION.md deleted file mode 100644 index 5dcc93d..0000000 --- a/CONFIGURATION.md +++ /dev/null @@ -1,53 +0,0 @@ -# Configuration (portable-python.yml) - -portable-python is configured via a YAML file (default: `portable-python.yml` in current directory, override with `--config`). - - -## Key Sections - -### folders - -- `build`, `dist`, `sources`, `logs`, `destdir`, `ppp-marker` -- All support path templates: `{build}`, `{version}`, `{abi_suffix}` - -### cpython-modules (CSV) - -Which external modules to auto-select. -Default is configured in `DEFAULT_CONFIG` (see `src/portable_python/config.py`). -Examples: openssl, zlib, xz, sqlite, bzip2, gdbm, libffi, readline, uuid - -### cpython-configure (list) - -Extra `./configure` args for CPython. -Default includes: `--enable-optimizations`, `--with-lto`, `--with-ensurepip=upgrade` - -### cpython-clean-1st-pass (list) - -Files to remove before `compileall` — removes test files, idle, 2to3 (~94 MB savings). - -### cpython-clean-2nd-pass (list) - -Files to remove after `compileall` — removes pycaches for seldom-used libs (~1.8 MB savings). - - -## Per-module config - -Each external module can be customized with these keys (replace `{module}` with the module name, e.g. `openssl`): - -| Key | Purpose | -|-----|---------| -| `{module}-version` | Version to use | -| `{module}-url` | URL to download from | -| `{module}-src-suffix` | File extension if not in URL | -| `{module}-configure` | Custom configure args | -| `{module}-http-headers` | HTTP headers for download | -| `{module}-patches` | File patches to apply | -| `{module}-debian` | Package name on Debian (for dependency detection) | - - -## Platform-specific overrides - -Configuration supports platform-specific sections (e.g. `windows.ext`, `macos.env`, etc.). -The most specific setting wins. - -`MACOSX_DEPLOYMENT_TARGET` defaults to 13 (Ventura). diff --git a/DEVELOP.md b/DEVELOP.md deleted file mode 100644 index 08ec5f1..0000000 --- a/DEVELOP.md +++ /dev/null @@ -1,111 +0,0 @@ -# Local development - -Create a dev venv: - -```shell -uv sync -``` - -You can then run `portable-python` from that venv: - -```shell -.venv/bin/portable-python list -.venv/bin/portable-python build-report 3.10.5 -``` - - -# Run the tests - -If you have tox, just run: `tox` to run all the tests. You can also run: -- `tox -e py313` to run with just one python version -- `tox -e style` to check PEP8 formatting -- etc - -If you don't have tox, you can run the tests with: `.venv/bin/pytest tests/` - -You can also run any of the `tests/` in IDEs such as PyCharm or VSCode. - -For example in PyCharm, just make sure that `pytest` is selected as "Default test runner" -in Preferences -> Tools -> Python Integrated Tools. -Then right-click on any file in `tests/` -(or in any function `test_...` function within a `test_*` file) -and select "Debug pytest in ..." - -You can set breakpoints as well during such test runs. - - -# Running in the debugger - -You can easily run `portable-python` in a debugger. -In PyCharm for example, you would simply browse to `.venv/bin/portable-python` -then right-click and select "Debug portable-python". -You can then edit the build/run configuration in PyCharm, add some "Parameters" to it, -like for example `build-report 3.13.2`, and then set breakpoints wherever you like. - -There is a `--dryrun` mode that can come in very handy for rapid iterations. - - -# Building a linux binary via docker - -Build a docker image, for example using the provided sample `Dockerfile`: - -```shell -docker build -t portable-python-jammy . -``` - -Run the docker image, with a folder `/src/` mounted to point to: - -```shell -docker run -it -v./:/src/ portable-python-jammy /bin/bash -``` - -Now inside docker, you run a build: - -```shell -portable-python build 3.13.2 -``` - - -# Key Dependencies - -| Package | Version | Purpose | -|---------|---------|---------| -| click | <9 | CLI framework | -| pyyaml | <7 | Configuration parsing | -| requests | <3 | HTTP downloads | -| runez | <6 | Utilities (file ops, system, logging) | -| urllib3 | <3 | HTTP transport | -| pytest-cov | - | Coverage reporting (dev only) | - -**runez** is central: provides file ops (`ls_dir`, `touch`, `compress`/`decompress`), system info (platform detection), CLI decorators (`@click.group`), logging, and version handling (`Version`, `PythonSpec`). - - -# Common Tasks - -## Add a New External Module - -1. Create class in `src/portable_python/external/xcpython.py` extending `ModuleBuilder` -2. Set `m_name`, `m_telltale`, `m_debian`, `version` property -3. Implement `url` property (or override `_do_linux_compile()` / `_do_macos_compile()`) -4. Add to `Cpython.candidate_modules()` if it's a CPython sub-module -5. Add tests in `tests/test_setup.py` - -## Add a New Config Option - -1. Update `DEFAULT_CONFIG` in `src/portable_python/config.py` -2. Use `PPG.config.get_value("key")` to retrieve it in code -3. Add tests to `tests/test_setup.py` - -## Fix a Portability Issue - -1. Run `portable-python inspect ` to diagnose -2. If lib is being dynamically linked, add to module list or update isolation -3. Use `LibAutoCorrect.run()` logic (or extend it) to fix paths -4. Add test case to `tests/test_inspector.py` - -## Bump Python Support - -1. Update `pyproject.toml` classifiers and `requires-python` -2. Update `.github/workflows/tests.yml` matrix -3. Update `CPythonFamily.min_version` if needed -4. Run full test matrix with `tox` diff --git a/docs/architecture/build-setup.md b/docs/architecture/build-setup.md new file mode 100644 index 0000000..904bb27 --- /dev/null +++ b/docs/architecture/build-setup.md @@ -0,0 +1,29 @@ +--- +type: Class +title: BuildSetup +description: Orchestrates a build end to end — resolves the spec, builds external modules then CPython, validates, and compresses the result. +tags: [class, build, coordinator] +timestamp: 2026-06-23T00:00:00Z +--- + +# BuildSetup + +The build orchestrator (`__init__.py`). The [`build`](/cli/build.md) command — and library users — construct one from a python spec and call `compile()`: + +```python +from portable_python import BuildSetup + +BuildSetup("cpython:3.13.2").compile() +``` + +## Mental model + +A conductor: it compiles nothing itself, it sequences the pieces. `compile()` is the spine — it cleans the [build folders](/concepts/build-layout.md), enters a [`BuildContext`](/concepts/folder-masking.md) (macOS isolation), builds the [external modules](/modules/index.md) into `build/deps/`, then builds CPython through its [`PythonBuilder`](/architecture/python-builder.md) (a [`Cpython`](/architecture/cpython.md)), and finally compresses the install into `dist/`. **External libraries first, CPython last.** + +It owns the [`Folders`](/architecture/config.md) (resolved paths) and the python builder; shared config, target platform, and version family come from [`PPG`](/architecture/ppg.md). + +## Worth knowing + +- **Start reading at `compile()`** — it calls everything else in order. +- A `--prefix` makes the build **non-portable**; without it the result is relocatable (see [ppp-marker](/concepts/ppp-marker.md) and [portability](/concepts/portability.md)). +- It requires a full `X.Y.Z` spec — a bare `3.13` is rejected; `"latest"` (or empty) resolves to the newest known version. diff --git a/docs/architecture/config.md b/docs/architecture/config.md new file mode 100644 index 0000000..bce9ae1 --- /dev/null +++ b/docs/architecture/config.md @@ -0,0 +1,30 @@ +--- +type: Class +title: Config & Folders +description: Config loads and merges YAML configuration with platform-specific overrides; Folders resolves the templated build/dist/sources paths. +tags: [class, configuration, folders, yaml] +timestamp: 2026-06-23T00:00:00Z +--- + +# Config & Folders + +`Config` (`config.py`) loads, merges, and queries the YAML [configuration](/configuration/portable-python-yml.md). `Folders` (`versions.py`) turns its templated path settings into concrete filesystem paths. Both are reached through [`PPG`](/architecture/ppg.md). + +## Merge & precedence + +The mental model is layering. A built-in `DEFAULT_CONFIG` provides sane defaults; user files (default `portable-python.yml`, plus any `include:`d files) layer on top. Within a file, **platform-specific** sections override generic ones, and the most specific match for the [target platform](/architecture/ppg.md) wins: + +```yaml +ext: gz # generic default +windows: + ext: zip # used only when targeting windows +macos: + env: + MACOSX_DEPLOYMENT_TARGET: 13 +``` + +A lookup returns that most-specific value (with the option to ignore platform overrides). + +## Folders + +`Folders` resolves the templated `folders:` settings (placeholders like `{build}`, `{version}`, `{abi_suffix}`) into the concrete `build/`, `deps/`, `dist/`, … paths every component uses — including the [ppp-marker](/concepts/ppp-marker.md) install prefix. See [build layout](/concepts/build-layout.md) for the resulting tree. diff --git a/docs/architecture/cpython.md b/docs/architecture/cpython.md new file mode 100644 index 0000000..6d642bd --- /dev/null +++ b/docs/architecture/cpython.md @@ -0,0 +1,24 @@ +--- +type: Class +title: Cpython +description: The concrete PythonBuilder that compiles CPython — configure/make/install, optimization flags, and the finalize step that makes the install relocatable. +tags: [class, cpython, build, finalize] +timestamp: 2026-06-23T00:00:00Z +--- + +# Cpython + +The concrete [`PythonBuilder`](/architecture/python-builder.md) that compiles CPython (`cpython.py`); `CPythonFamily.get_builder()` returns it (see [`PPG`](/architecture/ppg.md)). Most of its work is two things: configuring the build, and finalizing the install so it's relocatable. + +## Configure + +CPython is configured purely through flags — never source patches (the [no-patches principle](/overview.md)). Defaults come from `cpython-configure` in [config](/configuration/portable-python-yml.md) (optimizations, LTO, ensurepip), plus computed flags that point the build at the statically-compiled [deps](/concepts/static-linking.md). + +## Finalize — making it relocatable + +After `make install`, a portable build's interpreter still thinks it lives at the placeholder [`/ppp-marker/{version}`](/concepts/ppp-marker.md). Finalizing turns it into a [portable](/concepts/portability.md) install: + +- **Trim** — remove tests / idle / 2to3 before byte-compiling, then prune seldom-used pycaches after (driven by the `cpython-clean-*` config). +- **Byte-compile** the standard library. +- **Relativize** — rewrite `bin/` shebangs and the absolute paths baked into `sysconfig` so everything resolves relative to the install's own location. +- **Verify** — sanity-check that `venv` works in the finished build. diff --git a/docs/architecture/index.md b/docs/architecture/index.md new file mode 100644 index 0000000..8f80a8d --- /dev/null +++ b/docs/architecture/index.md @@ -0,0 +1,30 @@ +# Architecture + +The core classes and how they collaborate during a build. +## Global state & configuration + +* [PPG](/architecture/ppg.md) - Global singleton holding config, target platform, and version families. +* [Config](/architecture/config.md) - Loads and merges YAML configuration; the `Folders` helper resolves build paths. + +## Build coordination + +* [BuildSetup](/architecture/build-setup.md) - Drives the overall compilation: resolve spec, select modules, build, validate, compress. +* [ModuleBuilder](/architecture/module-builder.md) - Abstract base for anything that gets compiled (external C libs and Python itself). +* [PythonBuilder](/architecture/python-builder.md) - `ModuleBuilder` specialization for python implementations. +* [Cpython](/architecture/cpython.md) - Concrete builder: CPython's configure/make/install, optimization, and finalization. + +## Validation + +* [PythonInspector](/architecture/python-inspector.md) - Validates the portability of a built (or any) python by checking shared-library dependencies and paths. + +## Collaboration at a glance + +``` +PPG (config, target, families) + └─ BuildSetup(python_spec) + ├─ Folders (resolved build/dist/... paths) + ├─ BuildContext (macOS folder masking, isolation) + └─ python_builder : Cpython (a PythonBuilder, a ModuleBuilder) + └─ modules : ModuleCollection + └─ candidates/selected : ModuleBuilder (Openssl, Zlib, ...) +``` diff --git a/docs/architecture/module-builder.md b/docs/architecture/module-builder.md new file mode 100644 index 0000000..cd3e44a --- /dev/null +++ b/docs/architecture/module-builder.md @@ -0,0 +1,26 @@ +--- +type: Class +title: ModuleBuilder +description: Abstract base for everything that gets compiled — external C libraries and CPython itself — providing the common download/configure/make/install flow and environment injection. +tags: [class, abstract, build, module] +timestamp: 2026-06-23T00:00:00Z +--- + +# ModuleBuilder + +The abstract base for everything the tool compiles (`__init__.py`). Both the [external C modules](/modules/index.md) and [`PythonBuilder`](/architecture/python-builder.md) (hence [`Cpython`](/architecture/cpython.md)) extend it, so every component is built the same way and lands in the same [build layout](/concepts/build-layout.md). + +## Mental model + +A subclass is mostly **declarative**: it sets a few `m_*` class attributes (name, [telltale](/concepts/telltale-detection.md) marker, Debian package, include subfolder) and provides a source `url` + `version`. The base class runs the rest of the flow — download → unpack → patch → configure → make → install — into the shared `build/deps/` prefix. Platform differences are isolated to `_do_linux_compile()` / `_do_macos_compile()`, dispatched on [`PPG.target`](/architecture/ppg.md). + +Two mechanisms recur everywhere and are worth understanding: + +- **Environment injection** (`xenv_*` methods) — each contributes one variable (CPATH, LDFLAGS, PATH, …) pointing the compiler at `build/deps/`. This is how a statically-built dependency gets found by the next build; see [static linking](/concepts/static-linking.md). +- **Per-module config overrides** — `cfg_*` helpers read `{module}-version`, `{module}-url`, `{module}-configure`, … from [configuration](/configuration/portable-python-yml.md), so any default can be overridden without code changes. + +`linker_outcome()` decides, per module and platform, whether it will be linked `static` / `shared`, is `absent`, or would `fail` — the verdict surfaced by [`build-report`](/cli/build-report.md). + +## Adding one + +Subclass `ModuleBuilder`, set the `m_*` attributes, implement `url` / `version` and `_do_linux_compile()` — see the [guide](/guides/add-an-external-module.md). diff --git a/docs/architecture/ppg.md b/docs/architecture/ppg.md new file mode 100644 index 0000000..9a6cde7 --- /dev/null +++ b/docs/architecture/ppg.md @@ -0,0 +1,21 @@ +--- +type: Class +title: PPG +description: Global singleton holding shared configuration, the target platform, and the registry of supported python version families. +tags: [class, global-state, singleton, versions] +timestamp: 2026-06-23T00:00:00Z +--- + +# PPG + +`PPG` ("**P**ortable **P**ython **G**lobals") is the global state holder (`versions.py`) — the one place the rest of the code reaches for three process-wide facts: the merged [configuration](/architecture/config.md), the target platform (OS/arch), and the registry of python version families. The CLI's `main` initializes it once; everything else reads from it. + +Like [`ppp-marker`](/concepts/ppp-marker.md), the name is a deliberately uncommon, greppable token — searching for `PPG` surfaces only its own usages, with no collisions in upstream CPython or third-party source. + +## Why a singleton + +The build depends on global facts — which config is active, what platform we target, which python versions exist — so centralizing them avoids threading that state through every constructor and gives the CLI one initialization point. Shared helpers hang off it too: resolving a python on `PATH`, building [`Folders`](/architecture/config.md) for a version, and expanding [telltale](/concepts/telltale-detection.md) markers against the target's system includes. + +## Version families + +A `VersionFamily` (e.g. `CPythonFamily`) lists the available versions and hands back the right builder ([`Cpython`](/architecture/cpython.md) for cpython). Versions are fetched lazily and cached — from `python.org/ftp`, or GitHub tags when configured — which is why [`list`](/cli/list.md) hits the network on first use. A family also pins the minimum supported version (non-EOL only). diff --git a/docs/architecture/python-builder.md b/docs/architecture/python-builder.md new file mode 100644 index 0000000..ea246ff --- /dev/null +++ b/docs/architecture/python-builder.md @@ -0,0 +1,25 @@ +--- +type: Class +title: PythonBuilder +description: A ModuleBuilder specialization for python implementations — adds module selection, the python install layout, and helpers to run the freshly built interpreter. +tags: [class, abstract, python, build] +timestamp: 2026-06-23T00:00:00Z +--- + +# PythonBuilder + +The abstract [`ModuleBuilder`](/architecture/module-builder.md) specialization for building a python interpreter (`__init__.py`); the concrete implementation is [`Cpython`](/architecture/cpython.md). + +## What it adds over ModuleBuilder + +Beyond the base compile flow, a `PythonBuilder` owns the **module selection** (which [external modules](/modules/index.md) to build) and can **run the freshly-built interpreter** — used during finalize and validation. + +## ModuleCollection + +A `PythonBuilder` owns a `ModuleCollection` that resolves which external modules actually get built — a distinction worth keeping straight: + +- **candidates** — every module this builder *could* compile. +- **selected** — the ones chosen for *this* build (config + `--modules` + auto-selection). This is what's actually compiled, not all candidates. +- **auto-selected** — modules force-included because the build can't succeed without them (each module's `auto_select_reason()`). + +It's what [`build-report`](/cli/build-report.md) renders. diff --git a/docs/architecture/python-inspector.md b/docs/architecture/python-inspector.md new file mode 100644 index 0000000..8cb6005 --- /dev/null +++ b/docs/architecture/python-inspector.md @@ -0,0 +1,27 @@ +--- +type: Class +title: PythonInspector +description: Validates the portability of a python installation by parsing the dynamic-library dependencies of every executable and .so, and reporting any non-portable references. +tags: [class, inspection, validation, portability] +timestamp: 2026-06-23T00:00:00Z +--- + +# PythonInspector + +`PythonInspector` (`inspector.py`) answers the question the whole project exists for: *is this python [portable](/concepts/portability.md), and if not, why not?* It powers the [`inspect`](/cli/inspect.md) command and the portability gate at the end of [`BuildSetup.compile()`](/architecture/build-setup.md). + +## How it works + +It walks every executable and `.so` in an installation, lists each one's dynamic-library dependencies (parsing `ldd` on Linux / `otool` on macOS), and classifies every reference: + +- **system / standard** library → fine; +- **relative / self** reference → fine (what a relocatable build should have); +- **absolute, non-system** reference → a portability problem. + +For a portable build, any problem fails the inspection (the report exposes a single "first problem" verdict that callers check). + +## Relativization + +The fix lives in `LibAutoCorrect`, which rewrites the absolute library paths (rpaths) baked into binaries so they resolve relative to the install — `patchelf` on Linux (an `$ORIGIN`-relative rpath), `install_name_tool` on macOS (`@rpath` / `@loader_path`). It runs **during** finalize (so the result passes the portable check) and is exposed standalone as [`lib-auto-correct`](/cli/lib-auto-correct.md). + +The same step also supports **non-portable** builds: with `--prefix` or `--enable-shared`, it additionally keeps the real prefix's `lib/` in the rpath, so a build pinned to a fixed location still finds its libraries. (CPython is built with a deliberately long rpath — unlike `chrpath`, `patchelf` can set a fresh rpath of any length, leaving room to rewrite it.) diff --git a/docs/cli/build-report.md b/docs/cli/build-report.md new file mode 100644 index 0000000..991b657 --- /dev/null +++ b/docs/cli/build-report.md @@ -0,0 +1,24 @@ +--- +type: CLI Command +title: build-report +description: Show the status of buildable modules — which will be auto-compiled, which are present on the system, and which would fail — without building anything. +tags: [cli, build, report, command] +timestamp: 2026-06-23T00:00:00Z +--- + +# build-report + +Show which [modules](/modules/index.md) would be compiled for a spec, and whether the selection can actually build on this host — **without** building. The fastest way to catch a "broken" module before committing to a full compile. + +``` +portable-python build-report [OPTIONS] [PYTHON_SPEC] +``` + +For each candidate module it shows the [telltale](/concepts/telltale-detection.md) status and the linker outcome — `static` (will be compiled), `shared` (system copy used), `absent`, or `failed` (can't build here, e.g. a `-`-sigil Debian dev package is present). `PYTHON_SPEC` defaults to the latest known version. + +# Examples + +```shell +portable-python build-report 3.13.2 +portable-python build-report 3.13.2 -m openssl,sqlite +``` diff --git a/docs/cli/build.md b/docs/cli/build.md new file mode 100644 index 0000000..f490909 --- /dev/null +++ b/docs/cli/build.md @@ -0,0 +1,30 @@ +--- +type: CLI Command +title: build +description: Build a portable (or --prefix) python binary from source for the given version spec. +tags: [cli, build, command] +timestamp: 2026-06-23T00:00:00Z +--- + +# build + +Build a portable python binary from source — constructs a [`BuildSetup`](/architecture/build-setup.md) and runs it. The output tarball lands in `dist/` (e.g. `dist/cpython-3.13.2-macos-arm64.tar.gz`; see [build layout](/concepts/build-layout.md)). + +``` +portable-python build [OPTIONS] PYTHON_SPEC +``` + +Worth knowing (`--help` for the full flag list): + +- The spec must be a **full** `X.Y.Z` (a bare `3.13` is rejected); `latest` resolves to the newest known version. +- `--prefix` builds for a fixed location and **disables portability** (see [ppp-marker](/concepts/ppp-marker.md)); without it you get a relocatable build. +- `--modules/-m` overrides which [external modules](/modules/index.md) are included (default: config + auto-selection). + +# Examples + +```shell +portable-python build 3.13.2 # portable, modules auto-selected +portable-python build 3.13.2 -m openssl,zlib,xz # restrict modules +portable-python build 3.13.2 --prefix /apps/python3.13 # non-portable, fixed location +portable-python --dryrun build 3.13.2 # show the plan, compile nothing +``` diff --git a/docs/cli/diagnostics.md b/docs/cli/diagnostics.md new file mode 100644 index 0000000..a103835 --- /dev/null +++ b/docs/cli/diagnostics.md @@ -0,0 +1,17 @@ +--- +type: CLI Command +title: diagnostics +description: Show system diagnostics (invoker python, platform info) alongside the active configuration. +tags: [cli, diagnostics, command] +timestamp: 2026-06-23T00:00:00Z +--- + +# diagnostics + +Print system diagnostics next to the resolved [configuration](/architecture/config.md). Reach for it when a build behaves unexpectedly and you want to confirm which config files and target platform are actually in effect. + +``` +portable-python diagnostics +``` + +It shows the invoker python and platform info on one side, and the merged config — plus the files it was loaded from — on the other. diff --git a/docs/cli/index.md b/docs/cli/index.md new file mode 100644 index 0000000..076c090 --- /dev/null +++ b/docs/cli/index.md @@ -0,0 +1,19 @@ +# CLI + +`portable-python` is a single-entry-point CLI (built with `click` via `runez.click`); every command lives in `cli.py`. + +``` +portable-python [GLOBAL OPTIONS] COMMAND [ARGS] +``` + +Two global options are worth knowing: `--config/-c` (which [config](/configuration/portable-python-yml.md) file to use) and `--dryrun/-n` (print what *would* happen without doing it — the fastest way to understand a build). `portable-python --help` lists the rest. The top-level group loads config (via [`PPG`](/architecture/ppg.md)) before any subcommand runs. + +## Commands + +* [build](/cli/build.md) - Build a portable python binary. +* [build-report](/cli/build-report.md) - Show which modules will be compiled and whether they can build. +* [inspect](/cli/inspect.md) - Check whether a python installation is portable. +* [list](/cli/list.md) - List available versions for a family. +* [diagnostics](/cli/diagnostics.md) - Show system diagnostics and active config. +* [recompress](/cli/recompress.md) - Re-compress an existing tarball/folder (compare sizes). +* [lib-auto-correct](/cli/lib-auto-correct.md) - Rewrite a build's lib references to relative paths. diff --git a/docs/cli/inspect.md b/docs/cli/inspect.md new file mode 100644 index 0000000..ce20c80 --- /dev/null +++ b/docs/cli/inspect.md @@ -0,0 +1,29 @@ +--- +type: CLI Command +title: inspect +description: Inspect a python installation for non-portable dynamic library usage, reporting any references that would break portability. +tags: [cli, inspect, portability, command] +timestamp: 2026-06-23T00:00:00Z +--- + +# inspect + +Inspect any python installation and report whether it is [portable](/concepts/portability.md) — powered by the [`PythonInspector`](/architecture/python-inspector.md). + +``` +portable-python inspect [OPTIONS] PATH +``` + +Worth knowing (`--help` for the full flag list): + +- It **exits non-zero** when it finds a portability problem, so it works as a CI gate. +- `PATH` can be the special value `invoker` to inspect the python running the CLI itself. +- `--prefix` says the build was pinned to a prefix (not portable), relaxing the check; `--verbose/-v` shows the full `.so` report. + +# Examples + +```shell +portable-python inspect /usr/bin/python3 # system python — usually NOT portable +portable-python inspect -v ~/versions/3.13.2 # full detail +portable-python inspect invoker # the python running this CLI +``` diff --git a/docs/cli/lib-auto-correct.md b/docs/cli/lib-auto-correct.md new file mode 100644 index 0000000..2fd3deb --- /dev/null +++ b/docs/cli/lib-auto-correct.md @@ -0,0 +1,24 @@ +--- +type: CLI Command +title: lib-auto-correct +description: Scan a python installation and rewrite executables/libraries to use relative paths — the same relativization the build applies internally, runnable standalone. +tags: [cli, lib-auto-correct, relativize, command] +timestamp: 2026-06-23T00:00:00Z +--- + +# lib-auto-correct + +Rewrite a python installation's executables and libraries to reference their dependencies by **relative** path — the same [`LibAutoCorrect`](/architecture/python-inspector.md) step the build runs internally, exposed standalone so you can iterate on relativization without a full rebuild. See [static linking](/concepts/static-linking.md) for why it's needed. + +``` +portable-python lib-auto-correct [OPTIONS] PATH +``` + +It is **dry-run by default** — it shows what it would rewrite; pass `--commit` to actually apply the changes. `--prefix` overrides the build prefix (default: the scanned python's own `sysconfig` prefix). + +# Examples + +```shell +portable-python lib-auto-correct ~/versions/3.13.2 # preview (dry-run) +portable-python lib-auto-correct --commit ~/versions/3.13.2 # apply +``` diff --git a/docs/cli/list.md b/docs/cli/list.md new file mode 100644 index 0000000..182688e --- /dev/null +++ b/docs/cli/list.md @@ -0,0 +1,24 @@ +--- +type: CLI Command +title: list +description: List the latest available versions for a python family (default cpython), optionally as JSON. +tags: [cli, list, versions, command] +timestamp: 2026-06-23T00:00:00Z +--- + +# list + +List the latest available versions for a [version family](/architecture/ppg.md) (default `cpython`). + +``` +portable-python list [OPTIONS] [FAMILY] +``` + +Versions are fetched lazily from the network (python.org, or GitHub tags when `cpython-use-github` is set) and cached, so the first run may pause briefly. `--json` emits machine-readable output. + +# Examples + +```shell +portable-python list +portable-python list cpython --json +``` diff --git a/docs/cli/recompress.md b/docs/cli/recompress.md new file mode 100644 index 0000000..f0f8444 --- /dev/null +++ b/docs/cli/recompress.md @@ -0,0 +1,23 @@ +--- +type: CLI Command +title: recompress +description: Re-compress an existing binary tarball or folder into another format, mainly to compare resulting sizes. +tags: [cli, recompress, command] +timestamp: 2026-06-23T00:00:00Z +--- + +# recompress + +Re-compress an existing build (folder or tarball) into a different format and print the before/after sizes — handy for comparing compression trade-offs. A utility, not part of a normal build. + +``` +portable-python recompress PATH EXT +``` + +`PATH` is an existing folder or tarball (resolved against the configured build/dist folders if relative); `EXT` is the target compression, limited to the platform's supported formats (`--help` lists them). + +# Examples + +```shell +portable-python recompress dist/cpython-3.13.2-linux-x86_64.tar.gz tar.zst +``` diff --git a/docs/concepts/build-layout.md b/docs/concepts/build-layout.md new file mode 100644 index 0000000..c914b15 --- /dev/null +++ b/docs/concepts/build-layout.md @@ -0,0 +1,49 @@ +--- +type: Concept +title: Build Folder Layout +description: Every component follows the same conventional layout under build/, with external libraries sharing a deps prefix and the finished binary landing in dist/. +tags: [build-layout, concept, folders] +timestamp: 2026-06-23T00:00:00Z +--- + +# Build Folder Layout + +Every component — external modules and CPython alike — is built using the same conventional folder layout. This uniformity is what lets a single [`ModuleBuilder`](/architecture/module-builder.md) framework drive every compile. Folder paths are configurable; the defaults are shown below. + +# Schema + +``` +build/ + sources/ # Downloaded source tarballs (downloaded once, reused) + openssl-3.5.6.tar.gz + Python-3.13.2.tar.xz + components/ # Each module is unpacked + compiled here + openssl/ + cpython/ + deps/ # Shared --prefix for ALL external modules + include/ # → injected into CPATH + lib/ lib64/ # → injected into LDFLAGS + bin/ # → injected into PATH + logs/ # Per-component build logs, ordered + 00-portable-python.log + 01-openssl.log + 02-cpython.log + ppp-marker/{version}/ # The built python (its baked-in prefix is /ppp-marker/{version}) +dist/ + cpython-3.13.2-macos-arm64.tar.gz # Ready-to-go portable tarball +``` + +## Key folders + +| Folder | Role | +|--------|------| +| `sources/` | Cache of downloaded tarballs. Survives if you point it outside `build/`. | +| `components/` | Scratch space where each component's source is unpacked and compiled. | +| `deps/` | The shared install prefix passed to every external module's `./configure --prefix=build/deps`. CPython finds these via injected env vars — see [static linking](/concepts/static-linking.md). | +| `logs/` | One log file per component, numbered so they sort in compile order. | +| `ppp-marker/{version}/` | The built python installation, under the [ppp-marker](/concepts/ppp-marker.md) prefix. | +| `dist/` | Final portable tarball(s). | + +## Configurability + +All of these paths are templated and overridable in [configuration](/configuration/portable-python-yml.md) under the `folders:` key, using placeholders like `{build}`, `{version}`, and `{abi_suffix}`. The [`Folders`](/architecture/config.md) helper resolves them. For example, the dev config keeps builds per-version with `build: "build/{family}-{version}{abi_suffix}"`. diff --git a/docs/concepts/folder-masking.md b/docs/concepts/folder-masking.md new file mode 100644 index 0000000..2dc3358 --- /dev/null +++ b/docs/concepts/folder-masking.md @@ -0,0 +1,26 @@ +--- +type: Concept +title: Folder Masking (macOS) +description: On macOS, /usr/local is temporarily masked with a RAM disk during the build so that CPython's configure cannot accidentally discover and dynamically link Homebrew libraries. +tags: [macos, folder-mask, concept, isolation] +timestamp: 2026-06-23T00:00:00Z +--- + +# Folder Masking (macOS) + +On macOS, `/usr/local` is historically where Homebrew and other package managers install libraries and headers. If CPython's `configure` discovers those during a build, it may **dynamically link** against them — silently breaking [portability](/concepts/portability.md), because the resulting binary would depend on libraries that won't exist on another machine. + +## The mask + +To prevent this, the build temporarily **masks** `/usr/local` with an empty RAM disk for the duration of the compile. With `/usr/local` appearing empty, `configure` falls back to the statically-compiled libraries in `build/deps/` (see [static linking](/concepts/static-linking.md)) instead of system ones. + +This is implemented by `FolderMask` and orchestrated by `BuildContext`: + +- `BuildContext.__enter__` mounts the mask (and any other isolation hacks) before compilation begins. +- `BuildContext.__exit__` / `cleanup` unmounts it afterward, restoring `/usr/local`. + +Because it is a RAM disk mounted over the path (not a deletion), the real `/usr/local` is untouched and reappears the moment the mask is removed. + +## Scope + +This mask applies to **macOS only**. On Linux, isolation is achieved through telltale-driven module selection and the injected `CPATH`/`LDFLAGS` pointing at `build/deps/` first. diff --git a/docs/concepts/index.md b/docs/concepts/index.md new file mode 100644 index 0000000..31c1c9c --- /dev/null +++ b/docs/concepts/index.md @@ -0,0 +1,10 @@ +# Concepts + +The domain ideas behind portable-python. Read these to understand *why* the build works the way it does. + +* [Portability](/concepts/portability.md) - What "portable" means here, and how it is validated. +* [Static linking](/concepts/static-linking.md) - Why dependencies are compiled statically and paths relativized. +* [Telltale detection](/concepts/telltale-detection.md) - How the tool decides whether a library must be compiled or already exists. +* [Folder masking](/concepts/folder-masking.md) - The macOS `/usr/local` RAM-disk mask that prevents accidental dynamic linking. +* [Build layout](/concepts/build-layout.md) - The `build/` and `dist/` folder structure shared by every component. +* [ppp-marker prefix](/concepts/ppp-marker.md) - The placeholder install prefix that fills in for `--prefix` and is relativized for portable builds. diff --git a/docs/concepts/portability.md b/docs/concepts/portability.md new file mode 100644 index 0000000..3a847d4 --- /dev/null +++ b/docs/concepts/portability.md @@ -0,0 +1,35 @@ +--- +type: Concept +title: Portability +description: A python installation is portable when it can be unpacked into any folder and run without depending on non-standard system shared libraries. +tags: [portability, concept, validation] +timestamp: 2026-06-23T00:00:00Z +--- + +# Portability + +A python installation is **portable** when it can be copied or unpacked into an arbitrary folder and used as-is, without: + +- An installer or post-install step. +- A dependency on shared libraries that may be absent (or a different version) on the target machine. +- Hard-coded absolute paths baked into binaries, scripts, or `sysconfig`. + +Portability is the central guarantee of this project. Two design choices deliver it: + +1. **[Static linking](/concepts/static-linking.md)** of dependencies, so the binary carries what it needs rather than loading it from the system at runtime. +2. **Path relativization**, so shebangs and `sysconfig` values resolve relative to the install folder instead of an absolute build-time prefix. + +## Portable vs `--prefix` builds + +The tool can produce two kinds of output: + +| Build kind | Description | Relocatable? | +|------------|-------------|--------------| +| Portable (default) | Statically linked, paths relativized | Yes — run from anywhere | +| `--prefix PATH` | Built to live at a fixed location (e.g. `/apps/python3.13`) | No — must live at `PATH` | + +See the [`build`](/cli/build.md) command for how to choose. + +## Validation + +Portability is not assumed — it is **verified**. The [`PythonInspector`](/architecture/python-inspector.md) walks every executable and `.so` file, parses their dynamic dependencies (`ldd` on Linux, `otool` on macOS), and classifies each reference as a system library, a relative reference, or a problem. The [`inspect`](/cli/inspect.md) command surfaces this report and fails if a non-portable reference is found. diff --git a/docs/concepts/ppp-marker.md b/docs/concepts/ppp-marker.md new file mode 100644 index 0000000..7bee80d --- /dev/null +++ b/docs/concepts/ppp-marker.md @@ -0,0 +1,24 @@ +--- +type: Concept +title: The ppp-marker Prefix +description: "ppp-marker" (portable python path marker) is the install prefix handed to CPython's configure — the real --prefix for non-portable builds, or a searchable placeholder that gets relativized for portable builds. +tags: [ppp-marker, prefix, concept, portability, relativize] +timestamp: 2026-06-23T00:00:00Z +--- + +# The ppp-marker Prefix + +`ppp-marker` stands for **portable python path marker**. It is the install prefix the build hands to CPython's `configure`, and its role depends on the build kind: + +- **Non-portable (`--prefix`) builds** — `ppp-marker` is simply the `--prefix` you asked for; the python is built to live there for real. +- **Portable builds** — there is no real install location, so `ppp-marker` is a *dummy* placeholder folder that fills in for `--prefix`. CPython is configured with `--prefix=/ppp-marker/{version}`, so the freshly built interpreter initially believes it lives at `/ppp-marker/{version}`. The [finalize step](/architecture/cpython.md) then rewrites those baked-in absolute paths to be relative (see [static linking](/concepts/static-linking.md) and [portability](/concepts/portability.md)), making the install relocatable. + +## Why "ppp" + +The `ppp` token is chosen to be **easy to search and impossible to collide with**: grepping the codebase for `ppp` turns up only these usages, and `ppp-marker` appears nowhere in upstream CPython source. That uniqueness is what lets the finalize step find and rewrite every occurrence of the placeholder prefix unambiguously. ([`PPG`](/architecture/ppg.md) — "Portable Python Globals" — is named on the same principle.) + +## Where it shows up + +- Configured via the `ppp-marker` key (default `/ppp-marker/{version}{abi_suffix}`) — see [configuration](/configuration/portable-python-yml.md) and [`Folders`](/architecture/config.md). +- On disk the install lands under it, at `build/ppp-marker/{version}/` — see [build layout](/concepts/build-layout.md). +- Visible in `--dryrun` output as `./configure --prefix=/ppp-marker/{version}`. diff --git a/docs/concepts/static-linking.md b/docs/concepts/static-linking.md new file mode 100644 index 0000000..e0bec04 --- /dev/null +++ b/docs/concepts/static-linking.md @@ -0,0 +1,31 @@ +--- +type: Concept +title: Static Linking +description: Dependencies are compiled statically into a shared deps prefix and linked into CPython, so the binary carries no runtime dependency on system shared libraries. +tags: [static-linking, concept, build] +timestamp: 2026-06-23T00:00:00Z +--- + +# Static Linking + +To make a binary [portable](/concepts/portability.md), its dependencies must travel with it rather than be loaded from the host at runtime. portable-python achieves this by compiling each external C library **statically**. + +## How it works + +1. Each [external module](/modules/index.md) is compiled with flags that disable shared output and enable static, position-independent code — typically `--enable-shared=no --enable-static=yes` (and `-fPIC` where needed). +2. All modules install into a single shared prefix, `build/deps/` (see [build layout](/concepts/build-layout.md)). +3. When CPython is compiled, environment variables injected by the [`ModuleBuilder`](/architecture/module-builder.md) point its `configure`/`make` at that prefix: + - `CPATH` → `build/deps/include` + - `LDFLAGS` → `build/deps/lib` (and `lib64`) + - `PKG_CONFIG_PATH`, `PATH`, `LD_LIBRARY_PATH` as needed. +4. CPython links those static archives into `libpython` and its extension `.so` files. + +The `xenv_*` methods on `ModuleBuilder` are the mechanism: each returns the value for one environment variable, and they are gathered just before `configure`/`make` runs. + +## No source patches + +Static linking is achieved **only** through `configure`/`make` flags — never by editing upstream source. This keeps the tool maintainable as upstream versions change. See the [guiding principles](/overview.md). + +## Relativization + +Static linking removes runtime library dependencies, but absolute *paths* can still leak into the install (shebang lines, `sysconfig` variables). The [`Cpython`](/architecture/cpython.md) finalize step rewrites these to be relative, and [`LibAutoCorrect`](/architecture/python-inspector.md) rewrites dynamic-library references in binaries to relative form (`@loader_path`/`$ORIGIN` style). Together these make the install relocatable. diff --git a/docs/concepts/telltale-detection.md b/docs/concepts/telltale-detection.md new file mode 100644 index 0000000..bb69ac7 --- /dev/null +++ b/docs/concepts/telltale-detection.md @@ -0,0 +1,49 @@ +--- +type: Concept +title: Telltale Detection +description: Modules check for marker header files to determine whether a library is already present on the system, which drives whether it is compiled statically, linked dynamically, or skipped. +tags: [telltale, concept, module-selection] +timestamp: 2026-06-23T00:00:00Z +--- + +# Telltale Detection + +Before compiling, the tool must decide, for each candidate [external module](/modules/index.md), whether the library is already available on the build host. It does this by looking for a **telltale** — a marker file (usually a development header) whose presence indicates the system has that library installed. + +## The `m_telltale` attribute + +Each [`ModuleBuilder`](/architecture/module-builder.md) subclass declares `m_telltale`, a path (or list of paths) using an `{include}` placeholder that is expanded against the target's system include directories: + +```python +class Openssl(ModuleBuilder): + m_telltale = "{include}/openssl/ssl.h" + +class Sqlite(ModuleBuilder): + m_telltale = ["{include}/sqlite3.h"] +``` + +`PPG.find_telltale()` expands the placeholder against each `target.sys_include` directory and returns the first existing path, or `None`. + +## Linker outcome + +The telltale result, combined with whether the module was *selected* and the platform, feeds `ModuleBuilder.linker_outcome()`, which classifies the module as one of: + +| Outcome | Meaning | +|---------|---------| +| `static` | Will be compiled and statically linked. | +| `shared` | Present on system, will be dynamically linked (only acceptable for non-portable / system libs). | +| `absent` | Not present and not selected — skipped. | +| `failed` | A configuration that cannot succeed (see Debian markers below). | + +## Debian markers (`m_debian`) + +On Linux, `m_debian` names the Debian dev package and a leading sigil encodes a build constraint: + +| Prefix | Meaning | Example | +|--------|---------|---------| +| `!` | Required to compile **at all** — build is broken without it. | `!zlib1g-dev`, `!libffi-dev` | +| `+` | Required when the module is **selected** for static compile. | `+libsqlite3-dev`, `+uuid-dev` | +| `-` | Presence of the dev package **breaks** static compilation. | `-libreadline-dev`, `-tk-dev` | +| (none) | Used for detection/reporting only. | `libgdbm-dev` | + +This is why [`build-report`](/cli/build-report.md) can warn that a module is "broken" on the current host before a build is even attempted. diff --git a/docs/configuration/index.md b/docs/configuration/index.md new file mode 100644 index 0000000..765b175 --- /dev/null +++ b/docs/configuration/index.md @@ -0,0 +1,7 @@ +# Configuration + +How portable-python is configured, and how settings are merged and overridden. + +* [portable-python.yml](/configuration/portable-python-yml.md) - The config file: sections, per-module keys, path templates, and platform overrides. + +Configuration is loaded and merged by [`Config`](/architecture/config.md). The built-in `DEFAULT_CONFIG` provides defaults; user files layer on top, with platform-specific sections winning over generic ones. \ No newline at end of file diff --git a/docs/configuration/portable-python-yml.md b/docs/configuration/portable-python-yml.md new file mode 100644 index 0000000..a07dd99 --- /dev/null +++ b/docs/configuration/portable-python-yml.md @@ -0,0 +1,82 @@ +--- +type: Configuration +title: portable-python.yml +description: The YAML configuration file controlling folders, module selection, configure flags, cleanup, and per-module overrides — with platform-specific sections. +tags: [configuration, yaml, config] +timestamp: 2026-06-23T00:00:00Z +--- + +# portable-python.yml + +portable-python is configured via a YAML file — by default `portable-python.yml` in the current directory, overridable with `--config`. The built-in `DEFAULT_CONFIG` (in `config.py`) supplies defaults; user files layer on top and may `include:` other files. Values are read through [`Config.get_value()`](/architecture/config.md). + +# Schema + +## Top-level keys + +| Key | Type | Purpose | +|-----|------|---------| +| `folders` | map | Build path templates (see below). | +| `ext` | string | Output archive extension (`gz` default; `zip` on windows). | +| `cpython-modules` | CSV/list | Which [external modules](/modules/index.md) to auto-select. | +| `cpython-configure` | list | Extra `./configure` args (default: `--enable-optimizations`, `--with-lto`, `--with-ensurepip=upgrade`). | +| `cpython-clean-1st-pass` | list | Files removed before byte-compiling (~94 MB: tests, idle, 2to3). | +| `cpython-clean-2nd-pass` | list | Pycaches pruned after byte-compiling (~1.8 MB). | +| `cpython-compile-all` | bool | Whether to byte-compile the stdlib. | +| `cpython-additional-packages` | list | Extra pip packages to install into the build. | +| `cpython-use-github` | bool | Fetch the available version list from GitHub tags instead of python.org. | +| `include` | path | Layer another config file (a leading `+` puts it in front). | + +## `folders` + +All support path templates: `{build}`, `{family}`, `{version}`, `{abi_suffix}`. + +| Key | Default | Role | +|-----|---------|------| +| `build` | `build` | Root build folder. | +| `destdir` | `{build}` | `make install` DESTDIR. | +| `dist` | `dist` | Final tarball folder. | +| `logs` | `{build}/logs` | Per-component logs. | +| `sources` | `build/sources` | Downloaded tarball cache. | +| `ppp-marker` | `/ppp-marker/{version}{abi_suffix}` | The [ppp-marker](/concepts/ppp-marker.md) placeholder install prefix. | + +See [build layout](/concepts/build-layout.md) for how these map onto disk. + +## Per-module keys + +Replace `{module}` with the lowercased module name (e.g. `openssl`): + +| Key | Purpose | +|-----|---------| +| `{module}-version` | Version to build. | +| `{module}-url` | Source URL (`$version` is substituted; may carry a `#sha256=…` fragment, verified on download). | +| `{module}-src-suffix` | Archive extension when the URL lacks one. | +| `{module}-configure` | **Replace** the default configure args (`$deps_lib` substituted). | +| `{module}-http-headers` | Headers for the download (env vars expanded). | +| `{module}-patches` | Source patches to apply before compiling. | +| `{module}-debian` | Debian package name for dependency detection. | + +## Platform-specific overrides + +Sections named for a platform (`linux`, `macos`, `windows`) override generic keys; the most specific match for `PPG.target` wins. `MACOSX_DEPLOYMENT_TARGET` defaults to `13` (Ventura). + +# Examples + +```yaml +include: +pp-dev.yml + +folders: + build: "build/{family}-{version}{abi_suffix}" # keep builds per-version + +cpython-modules: libffi zlib xz bzip2 openssl uuid sqlite + +# Override a dependency version and source +libffi-version: 3.3 +openssl-url: https://my-openssl-mirror/openssl-$version.tar.gz +openssl-http-headers: + - Authorization: Bearer ${GITHUB_TOKEN} + +# Per-platform configure args +macos: + openssl-configure: --with-terminfo-dirs=/usr/share/terminfo +``` diff --git a/docs/guides/add-a-config-option.md b/docs/guides/add-a-config-option.md new file mode 100644 index 0000000..b955985 --- /dev/null +++ b/docs/guides/add-a-config-option.md @@ -0,0 +1,20 @@ +--- +type: Playbook +title: Add a Config Option +description: Add a new configuration key by extending DEFAULT_CONFIG and reading it via PPG.config.get_value(). +tags: [playbook, configuration, contributing] +timestamp: 2026-06-23T00:00:00Z +--- + +# Add a Config Option + +## Steps + +1. **Declare the default** in `DEFAULT_CONFIG` (in `src/portable_python/config.py`) so the key always has a value. +2. **Read it** where needed with [`PPG.config.get_value("your-key")`](/architecture/config.md) — pass `by_platform=False` if it should ignore platform-specific overrides. +3. **Add tests** in `tests/test_setup.py` (the suite runs in `--dryrun`). + +## Notes + +- Values support platform-specific overrides (`linux:` / `macos:` / `windows:` sections); the most specific match for [`PPG.target`](/architecture/ppg.md) wins. See [configuration](/configuration/portable-python-yml.md). +- For a per-module key, follow the `{module}-*` convention (`{module}-version`, `{module}-configure`, …) that the [`ModuleBuilder`](/architecture/module-builder.md) `cfg_*` helpers already read. diff --git a/docs/guides/add-an-external-module.md b/docs/guides/add-an-external-module.md new file mode 100644 index 0000000..69ae7c2 --- /dev/null +++ b/docs/guides/add-an-external-module.md @@ -0,0 +1,59 @@ +--- +type: Playbook +title: Add an External Module +description: Add a new statically-linked C library to CPython by creating a ModuleBuilder subclass and wiring it into the candidate list. +tags: [playbook, modules, contributing] +timestamp: 2026-06-23T00:00:00Z +--- + +# Add an External Module + +To statically link a new C library into CPython, add a [`ModuleBuilder`](/architecture/module-builder.md) subclass. The existing [external modules](/modules/external-modules.md) are the best templates. + +## Steps + +1. **Create the class** in `src/portable_python/external/xcpython.py`: + + ```python + class Mylib(ModuleBuilder): + """See https://docs.python.org/3/library/mymodule.html""" + + m_telltale = "{include}/mylib.h" # marker if system already has it + m_debian = "+libmylib-dev" # Linux build constraint (see telltale detection) + + @property + def url(self): + return self.cfg_url(self.version) or f"https://example.org/mylib-{self.version}.tar.gz" + + @property + def version(self): + return self.cfg_version("1.2.3") # default, overridable via mylib-version config + + def c_configure_args(self): + if args := self.cfg_configure(self.deps_lib_dir, self.deps_lib64_dir): + yield args + else: + yield "--enable-shared=no" + yield "--enable-static=yes" + + def _do_linux_compile(self): + self.run_configure("./configure", self.c_configure_args()) + self.run_make() + self.run_make("install") + ``` + +2. **Set the declarative attributes**: `m_telltale` (see [telltale detection](/concepts/telltale-detection.md)), `m_debian` with the right sigil, and `m_include`/`m_build_cwd` if needed. + +3. **Implement source resolution**: `url` and `version` (using `cfg_url`/`cfg_version` so config overrides work). + +4. **Implement the compile**: `_do_linux_compile()`. macOS reuses it unless you add `_do_macos_compile()`. + +5. **Register it** in `Cpython.candidate_modules()` (in `cpython.py`) so it becomes a candidate. + +6. **Add tests** in `tests/test_setup.py` (the suite runs in `--dryrun`, so it asserts the planned commands without compiling). + +## Tips + +- Prefer config flags over source patches — see the [no-patches principle](/overview.md). +- Use `--dryrun` to confirm the configure/make commands look right before a real build. +- If the library is only needed when explicitly selected, leave `auto_select_reason()` unset; if a build cannot succeed without it, return a short reason there. diff --git a/docs/guides/build-a-portable-python.md b/docs/guides/build-a-portable-python.md new file mode 100644 index 0000000..adb35b4 --- /dev/null +++ b/docs/guides/build-a-portable-python.md @@ -0,0 +1,61 @@ +--- +type: Playbook +title: Build a Portable Python +description: Install portable-python, build a statically-linked python binary, and use it from any folder. +tags: [playbook, build, getting-started] +timestamp: 2026-06-23T00:00:00Z +--- + +# Build a Portable Python + +End-to-end: from installing the tool to running the binary it produces. + +## 1. Install the tool + +`portable-python` is a regular python CLI: + +```shell +pipx install portable-python # or: pickley install portable-python +portable-python --help +``` + +## 2. (Optional) Preview the build + +A dry run shows exactly what would happen — downloads, configure/make commands, and the selected [modules](/modules/index.md) — without compiling: + +```shell +portable-python --dryrun build 3.13.2 +``` + +Use [`build-report`](/cli/build-report.md) to confirm the module selection is buildable on this host: + +```shell +portable-python build-report 3.13.2 +``` + +## 3. Build + +```shell +cd some-temp-folder +portable-python build 3.13.2 +ls -l dist/cpython-3.13.2-*.tar.gz +``` + +This runs the full [`BuildSetup.compile()`](/architecture/build-setup.md) pipeline: external libraries first, then CPython, then finalize + validate, then compress into `dist/`. + +## 4. Use it + +The tarball is [portable](/concepts/portability.md) — unpack anywhere and run, no installer: + +```shell +tar -C ~/versions/ -xf dist/cpython-3.13.2-macos-arm64.tar.gz +~/versions/3.13.2/bin/python --version +``` + +## 5. Verify portability (optional) + +```shell +portable-python inspect ~/versions/3.13.2 +``` + +A clean exit means no non-portable dynamic-library references — see [`inspect`](/cli/inspect.md). diff --git a/docs/guides/bump-python-support.md b/docs/guides/bump-python-support.md new file mode 100644 index 0000000..4b9a5c8 --- /dev/null +++ b/docs/guides/bump-python-support.md @@ -0,0 +1,18 @@ +--- +type: Playbook +title: Bump Python Support +description: Update the supported CPython versions across project metadata, the CI matrix, and the minimum version floor. +tags: [playbook, versions, cpython, maintenance] +timestamp: 2026-06-23T00:00:00Z +--- + +# Bump Python Support + +portable-python deliberately tracks only the recent, non-EOL CPython versions (see [overview](/overview.md)). To move the supported window: + +## Steps + +1. **Project metadata** — update the classifiers and `requires-python` in `pyproject.toml`. +2. **CI matrix** — update the Python versions in `.github/workflows/tests.yml` (see [CI/CD](/guides/ci-cd.md)). +3. **Minimum version** — bump `CPythonFamily.min_version` (in `versions.py`) if the floor moved; it gates which versions [`list`](/cli/list.md) and [`build`](/cli/build.md) accept. See [`PPG`](/architecture/ppg.md). +4. **Verify** — run the full matrix with `tox` (see [local development](/guides/local-development.md)). diff --git a/docs/guides/ci-cd.md b/docs/guides/ci-cd.md new file mode 100644 index 0000000..d1ad8b3 --- /dev/null +++ b/docs/guides/ci-cd.md @@ -0,0 +1,31 @@ +--- +type: Reference +title: CI/CD +description: The GitHub Actions workflows — tests on every push/PR across the Python matrix, and trusted PyPI publishing on version tags. +tags: [ci, cd, github-actions, release, reference] +timestamp: 2026-06-23T00:00:00Z +--- + +# CI/CD + +Two GitHub Actions workflows live in `.github/workflows/`. Both drive [tox](/guides/local-development.md) through `uv` / `tox-uv`. + +## Tests (`tests.yml`) + +Runs on every push to `main` and every pull request targeting `main`, with `cancel-in-progress` concurrency so superseded runs are cancelled. + +| Job | What it does | +|-----|--------------| +| `test` | Matrix over Python `3.10`–`3.14` on `ubuntu-latest` (`fail-fast: false`). Runs `uvx --with tox-uv tox -e py`, then uploads coverage to Coveralls as a **parallel** build, flagged per version. | +| `coveralls-finish` | After all `test` jobs finish, signals Coveralls that the parallel build is complete. | +| `linters` | On Python `3.14`, runs `uvx --with tox-uv tox -e style` (ruff check + format diff). | + +## Release (`release.yml`) + +Triggers on pushing a tag matching `v[0-9]*`. The single `publish` job (Python `3.14`, GitHub environment `release`): + +1. `uvx --with tox-uv tox -e py,style` — full test + lint gate. +2. `uv build` — build the sdist + wheel. +3. `pypa/gh-action-pypi-publish` — publish to PyPI via **trusted publishing** (OIDC, `id-token: write` — no API token is stored). + +See [bump python support](/guides/bump-python-support.md) for keeping the test matrix in sync with supported versions. diff --git a/docs/guides/fix-a-portability-issue.md b/docs/guides/fix-a-portability-issue.md new file mode 100644 index 0000000..bce706e --- /dev/null +++ b/docs/guides/fix-a-portability-issue.md @@ -0,0 +1,18 @@ +--- +type: Playbook +title: Fix a Portability Issue +description: Diagnose a non-portable build with inspect, then fix it by selecting the right module or relativizing lib references. +tags: [playbook, portability, inspect, debugging] +timestamp: 2026-06-23T00:00:00Z +--- + +# Fix a Portability Issue + +When a built python is flagged as non-[portable](/concepts/portability.md) — a dynamic dependency on a non-system library, or an absolute path baked in: + +## Steps + +1. **Diagnose** — run [`portable-python inspect `](/cli/inspect.md) to see exactly which `.so` or executable carries the offending reference, via the [`PythonInspector`](/architecture/python-inspector.md). +2. **If a library is being dynamically linked** — make sure its [external module](/modules/index.md) is selected so it gets compiled statically, or tighten build isolation so `configure` can't discover the system copy. On macOS check the [folder mask](/concepts/folder-masking.md); see also [telltale detection](/concepts/telltale-detection.md). +3. **If an absolute path is baked in** — the [`LibAutoCorrect`](/architecture/python-inspector.md) logic rewrites lib references to relative form. Run [`lib-auto-correct`](/cli/lib-auto-correct.md) to reproduce/iterate on it standalone, and extend it if a new case isn't handled. +4. **Add a test case** in `tests/test_inspector.py`. diff --git a/docs/guides/index.md b/docs/guides/index.md new file mode 100644 index 0000000..697e7c8 --- /dev/null +++ b/docs/guides/index.md @@ -0,0 +1,11 @@ +# Guides + +Task-oriented playbooks for using and extending portable-python. + +* [Build a portable python](/guides/build-a-portable-python.md) - Install the tool, build a binary, and use it. +* [Add an external module](/guides/add-an-external-module.md) - Add a new C library to the static-link set. +* [Add a config option](/guides/add-a-config-option.md) - Introduce a new configuration key. +* [Fix a portability issue](/guides/fix-a-portability-issue.md) - Diagnose and correct a non-portable build. +* [Bump python support](/guides/bump-python-support.md) - Move the supported CPython version window. +* [Local development](/guides/local-development.md) - Dev venv, tests, dryrun, and building Linux binaries via Docker. +* [CI/CD](/guides/ci-cd.md) - The GitHub Actions test and release workflows. diff --git a/docs/guides/local-development.md b/docs/guides/local-development.md new file mode 100644 index 0000000..8eddc90 --- /dev/null +++ b/docs/guides/local-development.md @@ -0,0 +1,75 @@ +--- +type: Playbook +title: Local Development +description: Set up a dev venv, run the tests, use dryrun mode, and build Linux binaries via Docker. +tags: [playbook, development, testing, docker] +timestamp: 2026-06-23T00:00:00Z +--- + +# Local Development + +## Dev venv + +```shell +uv sync +.venv/bin/portable-python list +.venv/bin/portable-python build-report 3.13.2 +``` + +## Running the tests + +The suite mocks `runez.run()` and builds in `--dryrun` mode, so it asserts the *planned* commands without actually compiling. Target is 100% coverage. + +```shell +tox # all envs (py310-314, coverage, style) +tox -e py313 # one python version +tox -e style # lint check (ruff) +tox -e reformat # auto-fix formatting + +.venv/bin/pytest tests/ # directly +.venv/bin/pytest tests/test_build.py::test_build_rc -vv # a single test +``` + +CI runs `uvx --with tox-uv tox -e py`. + +Test fixtures: a `cli` fixture (from runez) drives the CLI, `conftest.py` forbids real HTTP calls (`GlobalHttpCalls.forbid()`), and sample configs for parsing tests live in `tests/sample-config*.yml`. + +## Dryrun mode + +`--dryrun` (`-n`) prints what would be done without doing it — invaluable for fast iteration and for understanding the build: + +```shell +portable-python --dryrun build 3.13.2 +``` + +## Debugging + +`portable-python` can be run under a debugger (e.g. PyCharm: debug `.venv/bin/portable-python` with parameters like `build-report 3.13.2`). Set breakpoints in the [core classes](/architecture/index.md) or in any `tests/` file. + +## Building a Linux binary via Docker + +Real Linux builds (not dry runs) are easiest in a container: + +```shell +docker build -t portable-python-jammy . +docker run -it -v./:/src/ portable-python-jammy /bin/bash +# inside the container: +portable-python build 3.13.2 +``` + +## Code style + +Ruff handles both linting and formatting — `tox -e style` checks, `tox -e reformat` auto-fixes. The key settings (line length, max McCabe complexity, numpy-style docstrings, and relaxed security `S` rules in tests) live in `pyproject.toml`. + +## Documentation + +The project's documentation is the OKF knowledge bundle in `docs/` (start at [the bundle index](/index.md)). Validate its structure and cross-links with `scripts/check_okf.py docs`. + +## Key dependencies + +| Package | Purpose | +|---------|---------| +| `click` | CLI framework. | +| `pyyaml` | [Configuration](/configuration/portable-python-yml.md) parsing. | +| `requests` / `urllib3` | HTTP downloads. | +| `runez` | Foundational utilities — file ops, system info, CLI decorators, logging, `Version`/`PythonSpec`. Check `runez` before reimplementing anything. | diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..8953514 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,30 @@ +--- +okf_version: "0.1" +--- + +# Portable Python — Knowledge Bundle + +A knowledge bundle (in [OKF](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md) format) describing `portable-python`: a CLI and Python library for +compiling statically-linked, relocatable CPython binaries from source. + +Start with the [Overview](/overview.md), then drill into the area you need. + +## Sections + +* [Overview](/overview.md) - What portable-python does, its guiding principles, and how a build flows end to end. +* [Concepts](/concepts/index.md) - Domain ideas: portability, static linking, telltale detection, folder masking, build layout. +* [Architecture](/architecture/index.md) - The core classes and how they collaborate during a build. +* [CLI](/cli/index.md) - The `portable-python` subcommands and their options. +* [External Modules](/modules/index.md) - The C libraries compiled and statically linked into CPython. +* [Configuration](/configuration/index.md) - The `portable-python.yml` file, its sections and overrides. +* [Guides](/guides/index.md) - Playbooks for common tasks: building, extending, local development. + +## Conventions + +* Concept IDs are file paths minus `.md` (e.g. `architecture/ppg`). +* Cross-links are bundle-relative, beginning with `/` (the bundle root is this `docs/` folder).* Change history lives in [log.md](/log.md). + +## Authority + +This bundle is the **authoritative** documentation for portable-python's internals, configuration, CLI, and development workflows. +The repository's `README.rst` remains the project's landing page (install, quick start, motivation). diff --git a/docs/log.md b/docs/log.md new file mode 100644 index 0000000..b650c3d --- /dev/null +++ b/docs/log.md @@ -0,0 +1,23 @@ +# Knowledge Bundle Update Log + +## 2026-06-23 + +* **Update**: Pulled two verified facts in from the project wiki — the relativization tools (`patchelf` / `install_name_tool`) and `--prefix` / `--enable-shared` support ([PythonInspector](/architecture/python-inspector.md)), and the `#sha256=` download-checksum URL fragment ([configuration](/configuration/portable-python-yml.md)). Skipped the wiki's `sources:` download-url design — it's an unimplemented draft. +* **Update**: Trimmed the CLI pages — dropped the per-command option tables (which mirror `--help`) for purpose + non-obvious gotchas + examples, pointing at `--help` for the full flag list. +* **Update**: Reshaped the six architecture `type: Class` pages — dropped the `# Schema` member tables (which mirrored code and risked drift) for a condensed mental-model + "worth knowing" framing; the code stays authoritative for the API. +* **Update**: Removed the `resource:` frontmatter field from all concept files — unrendered, unreferenced by any tooling, and brittle (GitHub URLs). Source is pointed to inline in the prose instead. The bundle now contains no `github.com` repo links. +* **Update**: Removed the `# Citations` sections from all concept files — they duplicated the `resource:` frontmatter and inline source mentions, and added brittle GitHub-URL boilerplate that nothing referenced. Source pointers now live only in `resource:` and inline prose. +* **Update**: Inverted the `docs/` ↔ `CLAUDE.md` relationship — `CLAUDE.md` is now a thin pointer into the bundle. Folded the test-harness and code-style notes into [local development](/guides/local-development.md), and removed all back-references to `CLAUDE.md` from `docs/`. +* **Update**: Promoted `docs/` to the authoritative documentation; folded in and removed the root `ARCHITECTURE.md`, `CONFIGURATION.md`, and `DEVELOP.md`. +* **Creation**: Added guides for [CI/CD](/guides/ci-cd.md), [adding a config option](/guides/add-a-config-option.md), [fixing a portability issue](/guides/fix-a-portability-issue.md), and [bumping python support](/guides/bump-python-support.md). +* **Update**: Dropped the default-version column from the [external modules reference](/modules/external-modules.md) to avoid drift; it now points at `cfg_version()` in source and [`build-report`](/cli/build-report.md). +* **Creation**: Added the [ppp-marker prefix](/concepts/ppp-marker.md) concept. +* **Update**: Centralized the `ppp-marker` explanation there; build layout, PPG, Cpython, and configuration now link to it instead of repeating it. +* **Creation**: Established the portable-python knowledge bundle in OKF format. +* **Creation**: Added the [Overview](/overview.md). +* **Creation**: Added the [Concepts](/concepts/index.md) section (portability, static linking, telltale detection, folder masking, build layout). +* **Creation**: Added the [Architecture](/architecture/index.md) section covering the core classes (`PPG`, `BuildSetup`, `ModuleBuilder`, `PythonBuilder`, `Cpython`, `Config`, `PythonInspector`). +* **Creation**: Added the [CLI](/cli/index.md) reference for all subcommands. +* **Creation**: Added the [External Modules](/modules/index.md) reference. +* **Creation**: Added the [Configuration](/configuration/index.md) section. +* **Creation**: Added the [Guides](/guides/index.md) section (building, adding a module, local development). diff --git a/docs/modules/external-modules.md b/docs/modules/external-modules.md new file mode 100644 index 0000000..2b14b2c --- /dev/null +++ b/docs/modules/external-modules.md @@ -0,0 +1,54 @@ +--- +type: Reference +title: External Modules Reference +description: The C libraries compiled and statically linked into CPython, with their telltale markers and Linux build constraints. +tags: [reference, modules, external, cpython] +timestamp: 2026-06-23T00:00:00Z +--- + +# External Modules Reference + +These are the candidate external modules for CPython, in the order declared by `Cpython.candidate_modules()`. Each is a [`ModuleBuilder`](/architecture/module-builder.md) that compiles a C library statically into the shared `build/deps/` prefix before CPython is built. + +Default versions are intentionally **not** listed here, to avoid drifting from the code: each module's default is the `cfg_version()` default in its source class (overridable via the `{module}-version` [config](/configuration/portable-python-yml.md) key). To see the versions that would actually be used for a build, run [`build-report`](/cli/build-report.md). + +# Schema + +| Module | CPython feature | Telltale (`{include}/…`) | Linux dev pkg | +|--------|-----------------|--------------------------|----------------| +| `LibFFI` | `ctypes` | `ffi.h`, `ffi/ffi.h` | `!libffi-dev` | +| `Zlib` | `zlib` | `zlib.h` | `!zlib1g-dev` | +| `Zstd` | compression (zstd) | `zstd.h` | `!libzstd-dev` | +| `Xz` | `lzma` | `lzma.h` | — | +| `Bzip2` | `bz2` | `bzlib.h` | — | +| `Readline` | `readline` | `readline/readline.h` | `-libreadline-dev` | +| `Openssl` | `ssl`, `hashlib` | `openssl/ssl.h` | — | +| `Sqlite` | `sqlite3` | `sqlite3.h` | `+libsqlite3-dev` | +| `Bdb` | `dbm.ndbm` | `dbm.h` | `libgdbm-compat-dev` | +| `Gdbm` | `dbm.gnu` | `gdbm.h` | `libgdbm-dev` | +| `Uuid` | `uuid` | `uuid/uuid.h` | `+uuid-dev` | +| `TkInter` | `tkinter` | `tk`, `tk.h` | `-tk-dev` | +| `Mpdec` | `decimal` | `mpdecimal.h` | `!libmpdec-dev` | + +## Linux dev-package sigils + +The leading character on the Debian package encodes a build constraint (see [telltale detection](/concepts/telltale-detection.md)): + +| Sigil | Meaning | +|-------|---------| +| `!` | Required to compile **at all** (`LibFFI`, `Zlib`, `Zstd`, `Mpdec`). | +| `+` | Required when the module is **selected** for static compile (`Sqlite`, `Uuid`). | +| `-` | Dev package's presence **breaks** static compilation (`Readline`, `TkInter`). | +| (none) | Detection/reporting only. | + +## Sub-modules and toolchain + +Some modules bundle their own dependencies as sub-modules (their `candidate_modules()`): + +- `Readline` → `Ncurses` (`ncursesw`). +- `TkInter` → `Tcl`, `Tk`, `Tix` (defined in `external/xtkinter.py`). +- A separate `Toolchain` builds `GettextTiny` to prevent `libintl` from being picked up out of `/usr/local` — complementing the macOS [folder mask](/concepts/folder-masking.md). + +## Per-module config + +Every module honors these config keys (replace `{module}` with the lowercased name, e.g. `openssl`): `{module}-version`, `{module}-url`, `{module}-src-suffix`, `{module}-configure`, `{module}-http-headers`, `{module}-patches`. See [configuration](/configuration/portable-python-yml.md). diff --git a/docs/modules/index.md b/docs/modules/index.md new file mode 100644 index 0000000..9201cf8 --- /dev/null +++ b/docs/modules/index.md @@ -0,0 +1,7 @@ +# External Modules + +The C libraries that get compiled statically and linked into CPython, so that the corresponding standard-library modules work in a [portable](/concepts/portability.md) build. + +* [External modules reference](/modules/external-modules.md) - The full table: default versions, telltales, and Linux build constraints. + +Each module is a [`ModuleBuilder`](/architecture/module-builder.md) subclass. The set of candidates CPython can link is declared in `Cpython.candidate_modules()`; which ones are actually built depends on config, the `--modules` flag, [telltale detection](/concepts/telltale-detection.md), and auto-selection. diff --git a/docs/overview.md b/docs/overview.md new file mode 100644 index 0000000..ccdaa9f --- /dev/null +++ b/docs/overview.md @@ -0,0 +1,62 @@ +--- +type: Overview +title: Portable Python Overview +description: A CLI and library that compiles statically-linked, relocatable CPython binaries from source, and validates their portability. +tags: [overview, cpython, portability, build] +timestamp: 2026-06-23T00:00:00Z +--- + +# Overview + +`portable-python` compiles CPython from source into a binary that is **statically linked** and **relocatable**: the resulting tarball can be unpacked into any folder and used immediately, with no installer and no dependency on system shared libraries. It targets **Linux and macOS** (Intel and Apple Silicon). Windows is not supported. + +It is both: + +- A **CLI** with one entry point, `portable-python` (see [CLI](/cli/index.md)). +- A **Python library** (`from portable_python import BuildSetup`) for driving builds or inspections programmatically. + +## What a build produces + +``` +dist/cpython-3.13.2-macos-arm64.tar.gz +``` + +Unpack and run, no installation step: + +```shell +tar -C ~/versions/ -xf dist/cpython-3.13.2-macos-arm64.tar.gz +~/versions/3.13.2/bin/python --version +``` + +## How a build flows + +1. **Resolve** the requested [`PythonSpec`](/architecture/build-setup.md) (family + version) and target platform via the [`PPG`](/architecture/ppg.md) singleton. +2. **Select** which [external modules](/modules/index.md) to compile, based on config and [telltale detection](/concepts/telltale-detection.md). +3. **Compile external C libraries** first (openssl, zlib, xz, …) into a shared `build/deps/` prefix — see [build layout](/concepts/build-layout.md). +4. **Compile CPython** ([`Cpython`](/architecture/cpython.md)), pointing its `configure`/`make` at `build/deps/` via injected environment variables (CPATH, LDFLAGS, …). +5. **Finalize**: clean test files, byte-compile, [relativize paths](/concepts/static-linking.md) so the install is relocatable. +6. **Validate** portability with the [`PythonInspector`](/architecture/python-inspector.md), then compress into `dist/`. + +The whole pipeline is coordinated by [`BuildSetup`](/architecture/build-setup.md). + +## Guiding principles + +- **One job, done well.** Compile a portable python, validate it really is portable, drop the result in `dist/`. +- **No source patches.** C compilation relies solely on `configure`/`make` flags (e.g. `--enable-shared=no`), never on modifying upstream source. +- **Validated builds.** A large part of the effort is the [`inspect`](/cli/inspect.md) capability that proves an installation is portable (and explains why if it isn't). +- **Only recent, non-EOL Pythons.** No historical support; older versions are dropped without notice as the tool evolves. +- **Pure Python, testable.** No shell scripts. 100% test coverage, a `--dryrun` mode for fast iteration, and the ability to run under a debugger. + +# Examples + +Dry-run a build to see exactly what would happen without doing it: + +```shell +portable-python --dryrun build 3.13.2 +``` + +Inspect any existing python for portability: + +```shell +portable-python inspect /usr/bin/python3 +``` diff --git a/requirements.txt b/requirements.txt index 12ab1aa..812d78a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,20 @@ # This file was autogenerated by uv via the following command: # uv pip compile pyproject.toml --universal -o requirements.txt --python-version 3.10 --group dev -certifi==2026.4.22 +certifi==2026.6.17 # via requests charset-normalizer==3.4.7 # via requests -click==8.3.3 +click==8.4.1 # via portable-python (pyproject.toml) colorama==0.4.6 ; sys_platform == 'win32' # via # click # pytest -coverage==7.13.5 +coverage==7.14.2 # via pytest-cov exceptiongroup==1.3.1 ; python_full_version < '3.11' # via pytest -idna==3.13 +idna==3.18 # via requests iniconfig==2.3.0 # via pytest @@ -26,13 +26,13 @@ pluggy==1.6.0 # pytest-cov pygments==2.20.0 # via pytest -pytest==9.0.3 +pytest==9.1.1 # via pytest-cov pytest-cov==7.1.0 # via portable-python (pyproject.toml:dev) pyyaml==6.0.3 # via portable-python (pyproject.toml) -requests==2.33.1 +requests==2.34.2 # via portable-python (pyproject.toml) runez==5.9.1 # via portable-python (pyproject.toml) @@ -42,7 +42,7 @@ tomli==2.4.1 ; python_full_version <= '3.11' # pytest typing-extensions==4.15.0 ; python_full_version < '3.11' # via exceptiongroup -urllib3==2.6.3 +urllib3==2.7.0 # via # portable-python (pyproject.toml) # requests diff --git a/scripts/check_okf.py b/scripts/check_okf.py new file mode 100755 index 0000000..b889b87 --- /dev/null +++ b/scripts/check_okf.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +""" +Check an OKF (Open Knowledge Format) knowledge bundle for conformance. + +OKF spec: https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md + +Verifies: + - every non-reserved .md file has a parseable YAML frontmatter block with a non-empty `type` + - index.md files carry no frontmatter, except the bundle-root index.md (which declares `okf_version`) + - log.md carries no frontmatter + - markdown links to local .md files (bundle-relative "/..." or relative "./...") resolve to real files + +Exits non-zero if any conformance error or broken local link is found. + +Usage: + scripts/check_okf.py [BUNDLE_DIR] # default: docs +""" + +import argparse +import collections +import os +import re +import sys + +# Matches markdown links ending in .md, with an optional #anchor (external http(s)/mailto links are skipped below) +LINK_RE = re.compile(r"\]\((?P[^)\s#]+\.md)(?:#[^)]*)?\)") +TYPE_RE = re.compile(r"^type:\s*(?P\S.*)$", re.M) + + +def frontmatter(text): + """Return the YAML frontmatter block (without the --- fences), or None if absent.""" + if not text.startswith("---\n"): + return None + + end = text.find("\n---", 4) + if end == -1: + return None + + return text[4:end] + + +def check_bundle(root): + """Yield human-readable problem strings for the OKF bundle rooted at `root`.""" + md_files = [] + for dirpath, _, filenames in os.walk(root): + for name in filenames: + if name.endswith(".md"): + md_files.append(os.path.join(dirpath, name)) + + md_files.sort() + present = {os.path.relpath(p, root).replace(os.sep, "/") for p in md_files} + types = collections.Counter() + problems = [] + + for path in md_files: + rel = os.path.relpath(path, root).replace(os.sep, "/") + name = os.path.basename(path) + with open(path, encoding="utf-8") as fh: + text = fh.read() + + fm = frontmatter(text) + if name == "index.md": + if rel == "index.md": # bundle root: the one index.md allowed to have frontmatter + if not fm or "okf_version" not in fm: + problems.append(f"{rel}: bundle-root index.md must declare okf_version") + elif fm is not None: + problems.append(f"{rel}: non-root index.md must not have frontmatter") + elif name == "log.md": + if fm is not None: + problems.append(f"{rel}: log.md must not have frontmatter") + elif fm is None: + problems.append(f"{rel}: missing frontmatter") + else: + m = TYPE_RE.search(fm) + if m: + types[m.group("type").strip()] += 1 + else: + problems.append(f"{rel}: missing non-empty 'type' field") + + for m in LINK_RE.finditer(text): + target = m.group("target") + if target.startswith(("http://", "https://", "mailto:")): + continue # external link, not ours to resolve + + if target.startswith("/"): + resolved = target.lstrip("/") + else: + resolved = os.path.normpath(os.path.join(os.path.dirname(rel), target)).replace(os.sep, "/") + + if resolved not in present: + problems.append(f"{rel}: broken link -> {target}") + + return md_files, types, problems + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("bundle", nargs="?", default="docs", help="Path to the bundle root (default: docs)") + args = parser.parse_args(argv) + + if not os.path.isdir(args.bundle): + sys.exit(f"Not a directory: {args.bundle}") + + md_files, types, problems = check_bundle(args.bundle) + + print(f"Bundle: {args.bundle}") + print(f"Markdown files: {len(md_files)}") + if types: + print("Concept types:") + for concept_type, count in sorted(types.items()): + print(f" {count:2d} {concept_type}") + + if problems: + print(f"\n{len(problems)} problem(s):") + for problem in problems: + print(f" ERROR {problem}") + return 1 + + print("\nOK: bundle is OKF-conformant, all local links resolve.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())