diff --git a/docs/customization.md b/docs/customization.md index eb2973ac0..d6e765cc3 100644 --- a/docs/customization.md +++ b/docs/customization.md @@ -66,7 +66,7 @@ resolver_dist: include_wheels: true include_sdists: false git_options: - submodules: true # Clone all submodules for git+https:// URLs + submodules: true # Clone all submodules for git-based source providers build_dir: directory name relative to sdist directory, defaults to an empty string, which means to use the sdist directory env: USE_FFMPEG: "0" @@ -103,10 +103,10 @@ the package. Templating is not supported here. ### Git submodules -When building packages from git repositories using `git+https://` URLs in your -requirements, you can configure git submodule handling using the `git_options` -settings. This is useful for packages that depend on external libraries included -as git submodules. +When using a git-based source provider (e.g. `pypi-git`, `github-tag-git`), +you can configure git submodule handling using the `git_options` settings. +This is useful for packages that depend on external libraries included as +git submodules. #### Enabling all submodules @@ -147,25 +147,6 @@ git_options: - "vendor/lib2" ``` -Git submodules are supported for packages specified as `git+https://` URLs in -bootstrap requirements, such as: - -```text -my-package @ git+https://github.com/example/repo.git@v1.2.3 -``` - -Example requirements file with Git URLs: - -```{literalinclude} example/requirements-git-example.txt -:caption: requirements-git-example.txt -``` - -A complete package configuration example: - -```{literalinclude} example/git-submodules-example.yaml -:caption: git-submodules-example.yaml -``` - ### Build directory A `build_dir` field can also be defined to indicate to fromager where the @@ -262,16 +243,8 @@ dependency hooks (`get_build_backend_dependencies`, `get_build_sdist_dependencies`, etc.) that run *after* version resolution. -It is **not** available: - -- During the `resolve` phase itself — the version has not yet been - determined. -- When bootstrapping from a **git URL whose reference is not a valid - PEP 440 version** (for example - `pkg @ git+https://host/repo.git` or - `pkg @ git+https://host/repo.git@main`). In this case fromager - must build the package metadata just to discover the version, so - the early dependency-resolution hooks run with `version=None`. +It is **not** available during the `resolve` phase itself — the +version has not yet been determined. If your env var is used in a phase where the version might be unknown, add a fallback default so the substitution does not fail: @@ -563,7 +536,6 @@ def mycommand( **How-To Guides:** - [Override pyproject.toml](how-tos/pyproject-overrides.rst) - Modify build configuration -- [Build from Git](how-tos/build-from-git-repo.rst) - Use custom source repositories - [Multiple Versions](how-tos/multiple-versions.rst) - Handle version conflicts **Concepts:** diff --git a/docs/example/git-submodules-example.yaml b/docs/example/git-submodules-example.yaml deleted file mode 100644 index e826767b4..000000000 --- a/docs/example/git-submodules-example.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Example package configuration with git submodules support -# This file shows how to configure git submodule cloning for packages -# built from git repositories. - -# Enable submodules for all git URLs for this package -git_options: - submodules: true - -# Alternative: only clone specific submodule paths -# git_options: -# submodule_paths: -# - vendor/libfoo -# - vendor/libbar - -# You can also combine both settings: -# git_options: -# submodules: true -# submodule_paths: -# - vendor/lib1 -# - vendor/lib2 -# In this case, submodule_paths takes precedence and only the specified paths will be cloned. diff --git a/docs/example/requirements-git-example.txt b/docs/example/requirements-git-example.txt deleted file mode 100644 index 882a8c7af..000000000 --- a/docs/example/requirements-git-example.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Example requirements.txt with git submodules support -# This file shows how to specify git+https:// URLs in requirements.txt -# that can be configured with git submodules support - -# Standard PyPI package -pydantic-core==2.18.4 - -# Git repository URL with tag (configure git_options in package settings) -my-package @ git+https://github.com/example/repo.git@v1.2.3 - -# Git repository URL with branch -dev-package @ git+https://github.com/example/dev-repo.git@main - -# Git repository URL with commit hash -fixed-package @ git+https://github.com/example/fixed-repo.git@abc123def456 diff --git a/docs/how-tos/build-from-git-repo.rst b/docs/how-tos/build-from-git-repo.rst deleted file mode 100644 index d62dcd2ed..000000000 --- a/docs/how-tos/build-from-git-repo.rst +++ /dev/null @@ -1,55 +0,0 @@ -Building from a Git Repository -============================== - -You can build a package by cloning a git repository specifying the URL as part -of the requirement when bootstrapping. - -For example, if you want to build a package from the ``stevedore`` repository, -you can do the following: - -.. code-block:: console - - $ fromager bootstrap "stevedore @ git+https://github.com/openstack/stevedore.git" - -This will clone the ``stevedore`` repository and build the package from the -local copy. - -You can also use the ``bootstrap-parallel`` command for faster builds: - -.. code-block:: console - - $ fromager bootstrap-parallel "stevedore @ git+https://github.com/openstack/stevedore.git" - -This will perform the same operation but build wheels in parallel after the -bootstrap phase completes. - -Building from a specific version --------------------------------- - -To build a package from a git repository with a specific version, you can use -the ``@`` syntax to specify the version. - -.. code-block:: console - - $ fromager bootstrap "stevedore @ git+https://github.com/openstack/stevedore.git@5.2.0" - -This will clone the ``stevedore`` repository at the tag ``5.2.0`` and build the -package from the local copy. - -Or with parallel builds: - -.. code-block:: console - - $ fromager bootstrap-parallel "stevedore @ git+https://github.com/openstack/stevedore.git@5.2.0" - -.. important:: - - Building from a git repository URL is a special case which bypasses all of - fromager's resolver behavior (builtin and plugins) for the package. Other - plugins and override settings, such as preparing the source, patching it, - and building the wheel, are honored. - -.. important:: - - Git URL syntax is only supported in the top level requirements input file or - on the command line. Packages may not express dependencies using git URLs. diff --git a/docs/how-tos/index.rst b/docs/how-tos/index.rst index be6d9c62e..f411b3f08 100644 --- a/docs/how-tos/index.rst +++ b/docs/how-tos/index.rst @@ -33,7 +33,6 @@ Guides for building packages from various sources and configurations. .. toctree:: :maxdepth: 1 - build-from-git-repo repeatable-builds parallel build-web-server diff --git a/e2e/ci_bootstrap_parallel_suite.sh b/e2e/ci_bootstrap_parallel_suite.sh index 420514438..95821fb18 100755 --- a/e2e/ci_bootstrap_parallel_suite.sh +++ b/e2e/ci_bootstrap_parallel_suite.sh @@ -13,7 +13,5 @@ init_suite "Bootstrap Parallel Test Suite" "bootstrap-parallel tests" # Bootstrap parallel tests test_section "bootstrap parallel tests" run_test "bootstrap_parallel" -run_test "bootstrap_parallel_git_url" -run_test "bootstrap_parallel_git_url_tag" finish_suite diff --git a/e2e/ci_bootstrap_suite.sh b/e2e/ci_bootstrap_suite.sh index 5b5f874ce..5985db159 100755 --- a/e2e/ci_bootstrap_suite.sh +++ b/e2e/ci_bootstrap_suite.sh @@ -44,9 +44,4 @@ run_test "bootstrap_cooldown_github" run_test "bootstrap_cooldown_override" run_test "bootstrap_cooldown_prebuilt" -test_section "bootstrap git URL tests" -run_test "bootstrap_git_url" -run_test "bootstrap_git_url_tag" -run_test "version_env_git_url" - finish_suite diff --git a/e2e/ci_build_suite.sh b/e2e/ci_build_suite.sh index 344169c3e..a0b270d16 100755 --- a/e2e/ci_build_suite.sh +++ b/e2e/ci_build_suite.sh @@ -18,6 +18,5 @@ run_test "build_steps" test_section "advanced build tests" run_test "build_parallel" -run_test "build_sequence_git_url" finish_suite diff --git a/e2e/test_bootstrap_git_url.sh b/e2e/test_bootstrap_git_url.sh deleted file mode 100755 index 6f5897bcd..000000000 --- a/e2e/test_bootstrap_git_url.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# Test bootstrapping from a requirement with a git+https URL witout specifying a -# version tag. - -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source "$SCRIPTDIR/common.sh" - -GIT_REPO_URL="https://github.com/python-wheel-build/stevedore-test-repo.git" - -fromager \ - --debug \ - --log-file="$OUTDIR/bootstrap.log" \ - --error-log-file="$OUTDIR/fromager-errors.log" \ - --sdists-repo="$OUTDIR/sdists-repo" \ - --wheels-repo="$OUTDIR/wheels-repo" \ - --work-dir="$OUTDIR/work-dir" \ - --settings-dir="$SCRIPTDIR/changelog_settings" \ - bootstrap "stevedore @ git+${GIT_REPO_URL}" - -find "$OUTDIR/wheels-repo/" -name '*.whl' -find "$OUTDIR/sdists-repo/" -name '*.tar.gz' -ls "$OUTDIR"/work-dir/*/build.log || true - -EXPECTED_FILES=" -$OUTDIR/wheels-repo/downloads/setuptools-*.whl -$OUTDIR/wheels-repo/downloads/pbr-*.whl -$OUTDIR/wheels-repo/downloads/stevedore-*.whl - -$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz -$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz - -$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz -$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz -$OUTDIR/sdists-repo/builds/pbr-*.tar.gz - -$OUTDIR/work-dir/build-order.json -$OUTDIR/work-dir/constraints.txt - -$OUTDIR/bootstrap.log -$OUTDIR/fromager-errors.log - -$OUTDIR/work-dir/pbr-*/build.log -$OUTDIR/work-dir/setuptools-*/build.log -$OUTDIR/work-dir/stevedore-*/build.log -" - -pass=true -for pattern in $EXPECTED_FILES; do - if [ ! -f "${pattern}" ]; then - echo "Did not find $pattern" 1>&2 - pass=false - fi -done - -$pass - -twine check $OUTDIR/sdists-repo/builds/*.tar.gz -twine check $OUTDIR/wheels-repo/downloads/*.whl diff --git a/e2e/test_bootstrap_git_url_tag.sh b/e2e/test_bootstrap_git_url_tag.sh deleted file mode 100755 index 271e8ee3b..000000000 --- a/e2e/test_bootstrap_git_url_tag.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# Test bootstrapping from a requirement with a git+https URL and specifying a -# version tag. - -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source "$SCRIPTDIR/common.sh" - -GIT_REPO_URL="https://github.com/python-wheel-build/stevedore-test-repo.git" - -fromager \ - --debug \ - --log-file="$OUTDIR/bootstrap.log" \ - --error-log-file="$OUTDIR/fromager-errors.log" \ - --sdists-repo="$OUTDIR/sdists-repo" \ - --wheels-repo="$OUTDIR/wheels-repo" \ - --work-dir="$OUTDIR/work-dir" \ - --settings-dir="$SCRIPTDIR/changelog_settings" \ - bootstrap "stevedore @ git+${GIT_REPO_URL}@5.2.0" - -find "$OUTDIR/wheels-repo/" -name '*.whl' -find "$OUTDIR/sdists-repo/" -name '*.tar.gz' -ls "$OUTDIR"/work-dir/*/build.log || true - -EXPECTED_FILES=" -$OUTDIR/wheels-repo/downloads/setuptools-*.whl -$OUTDIR/wheels-repo/downloads/pbr-*.whl -$OUTDIR/wheels-repo/downloads/stevedore-*.whl - -$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz -$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz - -$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz -$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz -$OUTDIR/sdists-repo/builds/pbr-*.tar.gz - -$OUTDIR/work-dir/build-order.json -$OUTDIR/work-dir/constraints.txt - -$OUTDIR/bootstrap.log -$OUTDIR/fromager-errors.log - -$OUTDIR/work-dir/pbr-*/build.log -$OUTDIR/work-dir/setuptools-*/build.log -$OUTDIR/work-dir/stevedore-*/build.log -" - -pass=true -for pattern in $EXPECTED_FILES; do - if [ ! -f "${pattern}" ]; then - echo "Did not find $pattern" 1>&2 - pass=false - fi -done - -$pass - -twine check $OUTDIR/sdists-repo/builds/*.tar.gz -twine check $OUTDIR/wheels-repo/downloads/*.whl diff --git a/e2e/test_bootstrap_parallel_git_url.sh b/e2e/test_bootstrap_parallel_git_url.sh deleted file mode 100755 index 09361ec24..000000000 --- a/e2e/test_bootstrap_parallel_git_url.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# Test bootstrapping from a requirement with a git+https URL witout specifying a -# version tag. - -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source "$SCRIPTDIR/common.sh" - -GIT_REPO_URL="https://github.com/python-wheel-build/stevedore-test-repo.git" - -fromager \ - --debug \ - --log-file="$OUTDIR/bootstrap.log" \ - --error-log-file="$OUTDIR/fromager-errors.log" \ - --sdists-repo="$OUTDIR/sdists-repo" \ - --wheels-repo="$OUTDIR/wheels-repo" \ - --work-dir="$OUTDIR/work-dir" \ - --settings-dir="$SCRIPTDIR/changelog_settings" \ - bootstrap-parallel "stevedore @ git+${GIT_REPO_URL}" - -find "$OUTDIR/wheels-repo/" -name '*.whl' -find "$OUTDIR/sdists-repo/" -name '*.tar.gz' -ls "$OUTDIR"/work-dir/*/build.log || true - -EXPECTED_FILES=" -$OUTDIR/wheels-repo/downloads/setuptools-*.whl -$OUTDIR/wheels-repo/downloads/pbr-*.whl -$OUTDIR/wheels-repo/downloads/stevedore-*.whl - -$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz -$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz - -$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz -$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz -$OUTDIR/sdists-repo/builds/pbr-*.tar.gz - -$OUTDIR/work-dir/build-order.json -$OUTDIR/work-dir/constraints.txt - -$OUTDIR/bootstrap.log -$OUTDIR/fromager-errors.log - -$OUTDIR/work-dir/pbr-*/build.log -$OUTDIR/work-dir/setuptools-*/build.log -$OUTDIR/work-dir/stevedore-*/build.log -" - -pass=true -for pattern in $EXPECTED_FILES; do - if [ ! -f "${pattern}" ]; then - echo "Did not find $pattern" 1>&2 - pass=false - fi -done - -$pass - -twine check "$OUTDIR"/sdists-repo/builds/*.tar.gz -twine check "$OUTDIR"/wheels-repo/downloads/*.whl diff --git a/e2e/test_bootstrap_parallel_git_url_tag.sh b/e2e/test_bootstrap_parallel_git_url_tag.sh deleted file mode 100755 index 0de75ca22..000000000 --- a/e2e/test_bootstrap_parallel_git_url_tag.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# Test bootstrapping from a requirement with a git+https URL and specifying a -# version tag. - -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source "$SCRIPTDIR/common.sh" - -GIT_REPO_URL="https://github.com/python-wheel-build/stevedore-test-repo.git" - -fromager \ - --debug \ - --log-file="$OUTDIR/bootstrap.log" \ - --error-log-file="$OUTDIR/fromager-errors.log" \ - --sdists-repo="$OUTDIR/sdists-repo" \ - --wheels-repo="$OUTDIR/wheels-repo" \ - --work-dir="$OUTDIR/work-dir" \ - --settings-dir="$SCRIPTDIR/changelog_settings" \ - bootstrap-parallel "stevedore @ git+${GIT_REPO_URL}@5.2.0" - -find "$OUTDIR/wheels-repo/" -name '*.whl' -find "$OUTDIR/sdists-repo/" -name '*.tar.gz' -ls "$OUTDIR"/work-dir/*/build.log || true - -EXPECTED_FILES=" -$OUTDIR/wheels-repo/downloads/setuptools-*.whl -$OUTDIR/wheels-repo/downloads/pbr-*.whl -$OUTDIR/wheels-repo/downloads/stevedore-*.whl - -$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz -$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz - -$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz -$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz -$OUTDIR/sdists-repo/builds/pbr-*.tar.gz - -$OUTDIR/work-dir/build-order.json -$OUTDIR/work-dir/constraints.txt - -$OUTDIR/bootstrap.log -$OUTDIR/fromager-errors.log - -$OUTDIR/work-dir/pbr-*/build.log -$OUTDIR/work-dir/setuptools-*/build.log -$OUTDIR/work-dir/stevedore-*/build.log -" - -pass=true -for pattern in $EXPECTED_FILES; do - if [ ! -f "${pattern}" ]; then - echo "Did not find $pattern" 1>&2 - pass=false - fi -done - -$pass - -twine check "$OUTDIR"/sdists-repo/builds/*.tar.gz -twine check "$OUTDIR"/wheels-repo/downloads/*.whl diff --git a/e2e/test_build_sequence_git_url.sh b/e2e/test_build_sequence_git_url.sh deleted file mode 100755 index b9aca258f..000000000 --- a/e2e/test_build_sequence_git_url.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# Test bootstrapping from a requirement with a git+https URL witout specifying a -# version tag. - -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source "$SCRIPTDIR/common.sh" - -GIT_REPO_URL="https://github.com/python-wheel-build/stevedore-test-repo.git" - -fromager \ - --log-file="$OUTDIR/bootstrap.log" \ - --error-log-file="$OUTDIR/fromager-errors.log" \ - --sdists-repo="$OUTDIR/sdists-repo" \ - --wheels-repo="$OUTDIR/wheels-repo" \ - --work-dir="$OUTDIR/work-dir" \ - --settings-dir="$SCRIPTDIR/changelog_settings" \ - bootstrap "stevedore @ git+${GIT_REPO_URL}" - -find "$OUTDIR/wheels-repo/" -name '*.whl' -find "$OUTDIR/sdists-repo/" -name '*.tar.gz' -ls "$OUTDIR"/work-dir/*/build.log || true - -# Clean up the work directory so we can test build-sequence -mv "$OUTDIR/work-dir/build-order.json" "$OUTDIR/" -rm -rf "$OUTDIR/work-dir/wheels-repo" -rm -rf "$OUTDIR/work-dir/sdists-repo" - -# Rebuild using the build-order file and build-sequence -log="$OUTDIR/build.log" -cat "$OUTDIR/build-order.json" | jq . | tee -a "$log" - -fromager \ - --debug \ - --log-file "$log" \ - --work-dir "$OUTDIR/work-dir" \ - --sdists-repo "$OUTDIR/sdists-repo" \ - --wheels-repo "$OUTDIR/wheels-repo" \ - --settings-dir="$SCRIPTDIR/changelog_settings" \ - build-sequence --force "$OUTDIR/build-order.json" - -find "$OUTDIR/wheels-repo/" -name '*.whl' -find "$OUTDIR/sdists-repo/" -name '*.tar.gz' -ls "$OUTDIR"/work-dir/*/build.log || true - -EXPECTED_FILES=" -$OUTDIR/wheels-repo/downloads/setuptools-*.whl -$OUTDIR/wheels-repo/downloads/pbr-*.whl -$OUTDIR/wheels-repo/downloads/stevedore-*.whl - -$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz -$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz - -$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz -$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz -$OUTDIR/sdists-repo/builds/pbr-*.tar.gz - -$OUTDIR/bootstrap.log -$OUTDIR/build.log -$OUTDIR/fromager-errors.log - -$OUTDIR/work-dir/pbr-*/build.log -$OUTDIR/work-dir/setuptools-*/build.log -$OUTDIR/work-dir/stevedore-*/build.log -" - -pass=true -for pattern in $EXPECTED_FILES; do - if [ ! -f "${pattern}" ]; then - echo "Did not find $pattern" 1>&2 - pass=false - fi -done - -$pass - -twine check $OUTDIR/sdists-repo/builds/*.tar.gz -twine check $OUTDIR/wheels-repo/downloads/*.whl diff --git a/e2e/test_mode_build.sh b/e2e/test_mode_build.sh index 5f41dfe5f..8b799928a 100755 --- a/e2e/test_mode_build.sh +++ b/e2e/test_mode_build.sh @@ -2,7 +2,8 @@ # -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- # Test --test-mode with a package that fails to build (no prebuilt fallback) -# Uses a local fixture that fails during wheel build; since it's not on PyPI, +# Creates a local sdist from a fixture that intentionally fails during wheel +# build and serves it via the wheel server. Since the package is not on PyPI, # prebuilt fallback also fails and the failure is recorded. # See: https://github.com/python-wheel-build/fromager/issues/895 @@ -10,25 +11,45 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source "$SCRIPTDIR/common.sh" pass=true -DIST="test_build_failure" +DIST="test-build-failure" +VER="1.0.0" FIXTURE_DIR="$SCRIPTDIR/test_build_failure" -# Initialize the fixture as a git repo (files are committed without .git) -created_git=false -if [ ! -d "$FIXTURE_DIR/.git" ]; then - created_git=true - (cd "$FIXTURE_DIR" && \ - git init -q && \ - git config user.email "test@example.com" && \ - git config user.name "Test User" && \ - git add -A && \ - git commit -q -m "init") -fi +# Build a sdist tarball from the fixture +SDIST_NAME="test_build_failure-${VER}" +SDIST_TARBALL="${SDIST_NAME}.tar.gz" +SDIST_STAGING=$(mktemp -d) +# Clean up staging dir on exit (common.sh already traps for on_exit) +trap 'rm -rf "$SDIST_STAGING"' EXIT + +mkdir -p "$SDIST_STAGING/$SDIST_NAME" +cp -r "$FIXTURE_DIR"/* "$SDIST_STAGING/$SDIST_NAME/" +cat > "$SDIST_STAGING/$SDIST_NAME/PKG-INFO" << EOF +Metadata-Version: 2.1 +Name: test_build_failure +Version: ${VER} +Summary: Test fixture that intentionally fails to build +EOF +tar -czf "$SDIST_STAGING/$SDIST_TARBALL" -C "$SDIST_STAGING" "$SDIST_NAME" + +# Set up local index with the fixture tarball +# The wheel server basedir is wheels-repo/simple, so layout is: +# local-index/simple// +LOCAL_INDEX="$OUTDIR/local-index" +mkdir -p "$LOCAL_INDEX/simple/$DIST" +cp "$SDIST_STAGING/$SDIST_TARBALL" "$LOCAL_INDEX/simple/$DIST/" -# Clean up .git on exit if we created it -trap '[ "$created_git" = true ] && rm -rf "$FIXTURE_DIR/.git"' EXIT +# Start the wheel server to serve the local index +start_local_wheel_server "$LOCAL_INDEX" -echo "$DIST @ git+file://${FIXTURE_DIR}" > "$OUTDIR/requirements.txt" +# Configure fromager to resolve from local server +mkdir -p "$OUTDIR/settings" +cat > "$OUTDIR/settings/test_build_failure.yaml" << EOF +resolver_dist: + sdist_server_url: "${WHEEL_SERVER_URL}" + include_sdists: true + include_wheels: false +EOF set +e fromager \ @@ -36,7 +57,8 @@ fromager \ --sdists-repo="$OUTDIR/sdists-repo" \ --wheels-repo="$OUTDIR/wheels-repo" \ --work-dir="$OUTDIR/work-dir" \ - bootstrap --test-mode -r "$OUTDIR/requirements.txt" + --settings-dir="$OUTDIR/settings" \ + bootstrap --test-mode "${DIST}==${VER}" exit_code=$? set -e @@ -51,13 +73,14 @@ if [ -z "$failures_file" ]; then echo "FAIL: no test-mode-failures JSON file found" 1>&2 pass=false else - if ! jq -e ".failures[] | select(.package == \"$DIST\")" "$failures_file" >/dev/null 2>&1; then - echo "FAIL: $DIST not found in failures" 1>&2 + # The package name in the JSON is the canonicalized name + if ! jq -e '.failures[] | select(.package == "test-build-failure")' "$failures_file" >/dev/null 2>&1; then + echo "FAIL: test-build-failure not found in failures" 1>&2 pass=false fi # Must be 'bootstrap' failure (actual build failure, not resolution) - failure_type=$(jq -r "[.failures[] | select(.package == \"$DIST\")][0].failure_type" "$failures_file") + failure_type=$(jq -r '[.failures[] | select(.package == "test-build-failure")][0].failure_type' "$failures_file") if [ "$failure_type" != "bootstrap" ]; then echo "FAIL: expected failure_type 'bootstrap', got '$failure_type'" 1>&2 pass=false diff --git a/e2e/test_version_env_git_url.sh b/e2e/test_version_env_git_url.sh deleted file mode 100755 index 332245dab..000000000 --- a/e2e/test_version_env_git_url.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- - -# Test that ${__version__} in env settings fails when bootstrapping from -# a git URL without a PEP 440 version tag, and succeeds when a fallback -# default is provided via ${__version__:-...}. - -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source "$SCRIPTDIR/common.sh" - -GIT_REPO_URL="https://github.com/python-wheel-build/stevedore-test-repo.git" - -pass=true - -# --- Part 1: ${__version__} WITHOUT default should fail --- - -echo "=== Part 1: expect failure with \${__version__} (no default) ===" - -if fromager \ - --log-file="$OUTDIR/bootstrap-no-default.log" \ - --error-log-file="$OUTDIR/fromager-errors-no-default.log" \ - --sdists-repo="$OUTDIR/sdists-repo" \ - --wheels-repo="$OUTDIR/wheels-repo" \ - --work-dir="$OUTDIR/work-dir" \ - --settings-dir="$SCRIPTDIR/version_env_settings_no_default" \ - bootstrap "stevedore @ git+${GIT_REPO_URL}" 2>&1; then - echo "FAIL: bootstrap with \${__version__} (no default) should have failed" 1>&2 - pass=false -else - echo "OK: bootstrap with \${__version__} (no default) failed as expected" - if grep -q "__version__" "$OUTDIR/fromager-errors-no-default.log" 2>/dev/null || \ - grep -q "__version__" "$OUTDIR/bootstrap-no-default.log" 2>/dev/null; then - echo "OK: error message mentions __version__" - else - echo "WARN: error log does not mention __version__; check logs manually" - fi -fi - -# --- Part 2: ${__version__:-unresolved} WITH default should succeed --- - -echo "=== Part 2: expect success with \${__version__:-unresolved} ===" - -rm -rf "$OUTDIR/work-dir" "$OUTDIR/sdists-repo" "$OUTDIR/wheels-repo" -mkdir -p "$OUTDIR/build-logs" - -fromager \ - --log-file="$OUTDIR/bootstrap-with-default.log" \ - --error-log-file="$OUTDIR/fromager-errors-with-default.log" \ - --sdists-repo="$OUTDIR/sdists-repo" \ - --wheels-repo="$OUTDIR/wheels-repo" \ - --work-dir="$OUTDIR/work-dir" \ - --settings-dir="$SCRIPTDIR/version_env_settings_with_default" \ - bootstrap "stevedore @ git+${GIT_REPO_URL}" - -EXPECTED_FILES=" -$OUTDIR/wheels-repo/downloads/stevedore-*.whl -$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz -" - -for pattern in $EXPECTED_FILES; do - if [ ! -f "${pattern}" ]; then - echo "FAIL: Did not find $pattern" 1>&2 - pass=false - fi -done - -$pass diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index 1f12a8a0f..9e2bcfb73 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -109,32 +109,12 @@ def resolve( Contains one item when return_all_versions=False, or all matching versions when return_all_versions=True. - Raises: - ValueError: If req contains a git URL and pre_built is False - (git URL source resolution must be handled by Bootstrapper) """ - # Determine pre_built if not specified (needed for cache key and URL guard) + # Determine pre_built if not specified (needed for cache key) if pre_built is None: pbi = self.ctx.package_build_info(req) pre_built = pbi.pre_built - # Check session cache BEFORE the git URL guard so that background - # threads can retrieve pre-cached git URL resolutions (populated by - # Bootstrapper.resolve_versions() on the main thread before bootstrap() - # is called) without hitting the ValueError. - cached_result = self.get_cached_resolution(req, pre_built) - if cached_result is not None: - logger.debug(f"resolved {req} from cache") - return list(cached_result) if return_all_versions else [cached_result[0]] - - # Git URL source resolution must be handled by Bootstrapper. - # But git URL prebuilt resolution is allowed - we look for wheels on PyPI - # (test mode fallback uses this path). - if req.url and not pre_built: - raise ValueError( - f"Git URL requirements must be handled by Bootstrapper: {req}" - ) - rule_key = (str(req), pre_built) with self._lock: @@ -165,9 +145,6 @@ def get_cached_resolution( Returns ``None`` if the requirement has not been resolved yet, allowing callers to distinguish between "no matching versions" and "not yet resolved". - Used by background threads to retrieve pre-cached git URL resolutions - populated by the main thread before entering the parallel section. - Args: req: Package requirement pre_built: Whether looking for prebuilt or source resolution @@ -197,7 +174,7 @@ def _resolve_and_extend( parent_req=parent_req, ) - if cached_resolution and not req.url: + if cached_resolution: logger.debug( f"resolved from previous bootstrap: {len(cached_resolution)} version(s)" ) @@ -246,7 +223,12 @@ def _resolve_and_extend( ) if results: - self._extend_known_versions(req, pre_built, results) + key = (canonicalize_name(req.name), pre_built) + versions = self._known_versions.setdefault(key, {}) + for url, version in results: + if version not in versions or (url and not versions[version]): + versions[version] = url + self._resolved_rules.add((str(req), pre_built)) def _resolve_from_cache_server(self, req: Requirement) -> list[tuple[str, Version]]: """Fall back to the remote wheel cache server for a cached version. @@ -321,43 +303,6 @@ def _get_matching_versions( matching.sort(key=lambda x: x[1], reverse=True) return matching - def extend_known_versions( - self, - req: Requirement, - pre_built: bool, - result: list[tuple[str, Version]], - ) -> None: - """Extend the known-versions cache and mark the rule as resolved (thread-safe). - - Merges new versions into the package-level cache. When a version - already exists, a non-empty URL takes precedence over an empty one - (graph-resolved placeholders are replaced by real download URLs). - - Used by Bootstrapper to cache git URL resolutions that are - handled externally (outside this resolver). - - Args: - req: Package requirement (used for name and rule tracking) - pre_built: Whether this is a prebuilt or source resolution - result: List of (url, version) tuples to add - """ - with self._lock: - self._extend_known_versions(req, pre_built, result) - - def _extend_known_versions( - self, - req: Requirement, - pre_built: bool, - result: list[tuple[str, Version]], - ) -> None: - """Extend known versions (caller must hold ``self._lock``).""" - key = (canonicalize_name(req.name), pre_built) - versions = self._known_versions.setdefault(key, {}) - for url, version in result: - if version not in versions or (url and not versions[version]): - versions[version] = url - self._resolved_rules.add((str(req), pre_built)) - def _resolve_from_graph( self, req: Requirement, diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index f5773e531..bf864e41b 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -8,8 +8,6 @@ import logging import operator import pathlib -import shutil -import tempfile import time import typing @@ -19,9 +17,6 @@ from .. import ( bootstrap_requirement_resolver, - build_environment, - dependencies, - gitutils, progress, sources, ) @@ -220,11 +215,8 @@ def resolve_versions( ) -> list[tuple[str, Version]]: """Resolve version(s) of a requirement. - Returns list of (source URL, version) tuples, sorted by version (highest first). - - Git URL resolution stays in Bootstrapper because it requires - build orchestration (BuildEnvironment, build dependencies). - Delegates PyPI/graph resolution to BootstrapRequirementResolver. + Returns list of (source URL, version) tuples, sorted by version + (highest first). Delegates to ``BootstrapRequirementResolver``. Args: req: Package requirement to resolve @@ -237,29 +229,16 @@ def resolve_versions( Returns: List of (url, version) tuples. Contains one item when return_all_versions=False, or all matching versions when True. + + Raises: + ValueError: If ``req`` contains a direct-reference URL. """ if req.url: - if req_type != RequirementType.TOP_LEVEL: - raise ValueError( - f"{req} includes a URL, but is not a top-level dependency" - ) - - # Check cache first to avoid re-resolving - # Git URLs are always source (not prebuilt) - cached_result = self._resolver.get_matching_versions(req, pre_built=False) - if cached_result: - logger.debug(f"resolved {req} from cache") - return cached_result if return_all_versions else [cached_result[0]] - - logger.info("resolving source via URL, ignoring any plugins") - source_url, resolved_version = self._resolve_version_from_git_url(req=req) - # Cache the git URL resolution (always source, not prebuilt) - # Store as list for consistency with cache structure - result = [(source_url, resolved_version)] - self._resolver.extend_known_versions(req, pre_built=False, result=result) - return result # Git URLs always return single version - - # Delegate to RequirementResolver + raise ValueError( + f"{req.name}: PEP 508 direct-reference URLs (req.url) are no longer " + f"supported. Use a 'source' provider in the package settings instead. " + f"See https://fromager.readthedocs.io/en/latest/proposals/deprecate-req-url.html" + ) return self._resolver.resolve( req=req, req_type=req_type, @@ -461,111 +440,6 @@ def explain(self) -> str: for req_type, req, resolved_version in reversed(self.why) ) - def _prepare_build_dependencies( - self, - req: Requirement, - resolved_version: Version | None, - sdist_root_dir: pathlib.Path, - build_env: build_environment.BuildEnvironment, - ) -> set[Requirement]: - """Prepare build dependencies for a package. - - Only used by the git URL resolution path - (_resolve_version_from_git_url -> _get_version_from_package_metadata). - The main iterative bootstrap loop handles build deps via phase handlers. - """ - # build system - build_system_dependencies = dependencies.get_build_system_dependencies( - ctx=self.ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - ) - self._handle_build_requirements( - req, - RequirementType.BUILD_SYSTEM, - build_system_dependencies, - ) - # The next hooks need build system requirements. - build_env.install(build_system_dependencies) - - # build backend - build_backend_dependencies = dependencies.get_build_backend_dependencies( - ctx=self.ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - ) - - # build sdist - build_sdist_dependencies = dependencies.get_build_sdist_dependencies( - ctx=self.ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - ) - - # Filter out deps already satisfied by build-system dependencies - resolved_build_sys = self._resolve_build_system_versions_by_name( - build_system_dependencies, - ) - build_backend_dependencies = self.filter_deps_satisfied_by_build_system( - build_backend_dependencies, - resolved_build_sys, - RequirementType.BUILD_BACKEND, - ) - build_sdist_dependencies = self.filter_deps_satisfied_by_build_system( - build_sdist_dependencies, - resolved_build_sys, - RequirementType.BUILD_SDIST, - ) - - self._handle_build_requirements( - req, - RequirementType.BUILD_BACKEND, - build_backend_dependencies, - ) - self._handle_build_requirements( - req, - RequirementType.BUILD_SDIST, - build_sdist_dependencies, - ) - - build_dependencies = build_sdist_dependencies | build_backend_dependencies - if build_dependencies.isdisjoint(build_system_dependencies): - build_env.install(build_dependencies) - - return ( - build_system_dependencies - | build_backend_dependencies - | build_sdist_dependencies - ) - - def _handle_build_requirements( - self, - req: Requirement, - build_type: RequirementType, - build_dependencies: set[Requirement], - ) -> None: - """Bootstrap build dependencies. - - Only used by the git URL resolution path - (_resolve_version_from_git_url -> _get_version_from_package_metadata). - The main iterative bootstrap loop handles build deps via phase handlers. - """ - self.progressbar.update_total(len(build_dependencies)) - - for dep in self._sort_requirements(build_dependencies): - with req_ctxvar_context(dep): - # Save/restore self.why because the iterative bootstrap() - # modifies it internally for each work item. - saved_why = list(self.why) - self._bootstrap_one(req=dep, req_type=build_type) - self.why = saved_why - self.progressbar.update() - def _download_prebuilt( self, req: Requirement, @@ -687,148 +561,6 @@ def _handle_test_mode_failure( # Return None to signal failure; bootstrap() will record via re-raised exception return None - def _resolve_version_from_git_url(self, req: Requirement) -> tuple[str, Version]: - """Resolve source path and version from a ``git+`` URL. - - Parses the URL for an ``@ref`` version hint. If the ref is a valid - version, reuses an existing clone when possible. Otherwise, clones - the repo and extracts the version from package metadata. - """ - - if not req.url: - raise ValueError(f"unable to resolve from URL with no URL in {req}") - - # We start by not knowing where we would put the source because we don't - # know the version. - working_src_dir: pathlib.Path | None = None - version: Version | None = None - - url_to_clone, git_ref = gitutils.parse_vcs_url(req.url, require_ref=False) - need_to_clone = False - - if git_ref == gitutils.GIT_HEAD: - # No ref in URL, clone to discover the version. - logger.debug("no reference in URL, will clone") - need_to_clone = True - else: - # If we have a reference, it might be a valid python version - # string, or not. It _must_ be a valid git reference. If it can - # be parsed as a valid python version, we assume the tag points - # to source that will think that is its version, so we allow - # reusing an existing cloned repo if there is one. - try: - version = Version(git_ref) - except ValueError: - logger.info( - "could not parse %r as a version, cloning to get the version", - git_ref, - ) - need_to_clone = True - else: - logger.info("URL %s includes version %s", req.url, version) - working_src_dir = ( - self.ctx.work_dir - / f"{req.name}-{version}" - / f"{req.name}-{version}" - ) - if not working_src_dir.exists(): - need_to_clone = True - else: - if self.ctx.cleanup: - logger.debug("cleaning up %s to reclone", working_src_dir) - shutil.rmtree(working_src_dir) - need_to_clone = True - else: - logger.info("reusing %s", working_src_dir) - - if need_to_clone: - with tempfile.TemporaryDirectory() as tmpdir: - clone_dir = pathlib.Path(tmpdir) / "src" - sources.download_git_source( - ctx=self.ctx, - req=req, - url_to_clone=url_to_clone, - destination_dir=clone_dir, - ref=git_ref, - ) - if not version: - # If we still do not have a version, get it from the package - # metadata. - version = self._get_version_from_package_metadata(req, clone_dir) - logger.info("found version %s", version) - working_src_dir = ( - self.ctx.work_dir - / f"{req.name}-{version}" - / f"{req.name}-{version}" - ) - if working_src_dir.exists(): - # We have to check if the destination directory exists - # because if we were not given a version we did not - # clean it up earlier. We do not use ctx.cleanup to - # control this action because we cannot trust that the - # destination directory is reusable because we have had - # to compute the version and we cannot be sure that the - # version is dynamic. Two different commits in the repo - # could have the same version if that version is set - # with static data in the repo instead of via a tag or - # dynamically computed by something like setuptools-scm. - logger.debug("cleaning up %s", working_src_dir) - shutil.rmtree(working_src_dir) - assert working_src_dir is not None - working_src_dir.parent.mkdir(parents=True, exist_ok=True) - logger.info("moving cloned repo to %s", working_src_dir) - shutil.move(clone_dir, str(working_src_dir)) - - if not version: - raise ValueError(f"unable to determine version for {req}") - - if not working_src_dir: - raise ValueError(f"unable to determine working source directory for {req}") - - logging.info("resolved from git URL to %s, %s", working_src_dir, version) - return (str(working_src_dir), version) - - def _get_version_from_package_metadata( - self, - req: Requirement, - source_dir: pathlib.Path, - ) -> Version: - pbi = self.ctx.package_build_info(req) - build_dir = pbi.build_dir(source_dir) - - logger.info( - "preparing build dependencies so we can access the metadata to get the version" - ) - build_env = build_environment.BuildEnvironment( - ctx=self.ctx, - parent_dir=source_dir.parent, - ) - build_dependencies = self._prepare_build_dependencies( - req=req, - resolved_version=None, - sdist_root_dir=source_dir, - build_env=build_env, - ) - build_env.install(build_dependencies) - - logger.info("generating metadata to get version") - hook_caller = dependencies.get_build_backend_hook_caller( - ctx=self.ctx, - req=req, - build_dir=build_dir, - override_environ={}, - build_env=build_env, - ) - metadata_dir_base = hook_caller.prepare_metadata_for_build_wheel( - metadata_directory=str(source_dir.parent), - config_settings=pbi.config_settings, - ) - metadata_filename = source_dir.parent / metadata_dir_base / "METADATA" - # Disable validation because some packages have metadata version mismatches - # (e.g., declaring Metadata-Version: 2.2 but using fields from 2.4). - metadata = dependencies.parse_metadata(metadata_filename, validate=False) - return metadata.version - def add_to_graph( self, req: Requirement, @@ -970,8 +702,6 @@ def add_to_build_order( "source_url": source_url, "source_url_type": str(source_type), } - if req.url: - info["source_url"] = req.url self._build_stack.append(info) def _check_write_error(self, fut: concurrent.futures.Future[int]) -> None: diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 58bf2f6bd..2e33f8f0e 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -182,14 +182,7 @@ def build_sequence( resolved_version = Version(entry["version"]) source_download_url = entry["source_url"] - # If we are building from git, use the requirement as specified so - # we include the URL. Otherwise, create a fake requirement with the - # name and version so we are explicitly building the expected - # version. - if entry["source_url_type"] == "git": - req = Requirement(entry["req"]) - else: - req = Requirement(f"{dist_name}=={resolved_version}") + req = Requirement(f"{dist_name}=={resolved_version}") with req_ctxvar_context(req, resolved_version): logger.info("building %s", resolved_version) @@ -664,14 +657,7 @@ def update_progressbar_cb(future: concurrent.futures.Future) -> None: logger.info("requires exclusive build") logger.info("ready to build") - # Use original top-level requirement only if it has a URL (git, etc.), - # otherwise use the resolved version to avoid building wrong versions. - top_level_req = graph.get_top_level_requirement(node) - if top_level_req and top_level_req.url: - req = top_level_req - logger.debug("using top-level requirement with URL: %s", req) - else: - req = Requirement(f"{node.canonicalized_name}=={node.version}") + req = Requirement(f"{node.canonicalized_name}=={node.version}") future = executor.submit( _build_parallel, diff --git a/src/fromager/constraints.py b/src/fromager/constraints.py index 1d1530a52..d3f78c5d0 100644 --- a/src/fromager/constraints.py +++ b/src/fromager/constraints.py @@ -52,7 +52,7 @@ def add_constraint(self, unparsed: str) -> None: .. versionchanged: 0.83.0 Non-conflicting constraints are now combined. Constraints with conflicts now raise :exc:`InvalidConstraintError`. Inputs without a - version specifier or with extras or url are also refused. + version specifier or with extras are also refused. """ req = Requirement(unparsed) canon_name = canonicalize_name(req.name) @@ -62,7 +62,7 @@ def add_constraint(self, unparsed: str) -> None: if req.extras: raise InvalidConstraintError(f"Constraint {unparsed!r} has extras") if req.url: - raise InvalidConstraintError(f"Constraint {unparsed!r} has an url") + raise InvalidConstraintError(f"Constraint {unparsed!r} has a URL") if not req.specifier: raise InvalidConstraintError(f"Constraint {unparsed!r} has no specifiers") diff --git a/src/fromager/dependency_graph.py b/src/fromager/dependency_graph.py index 344580490..6d8aa3496 100644 --- a/src/fromager/dependency_graph.py +++ b/src/fromager/dependency_graph.py @@ -455,19 +455,6 @@ def get_nodes_by_name(self, req_name: str | None) -> list[DependencyNode]: def get_root_node(self) -> DependencyNode: return self.nodes[ROOT] - def get_top_level_requirement(self, node: DependencyNode) -> Requirement | None: - """Get the top-level requirement specification for a node. - - For packages that were specified as top-level requirements (e.g., with git URLs), - this returns the original requirement specification. Returns None if the node - is not a direct child of ROOT. - """ - root = self.get_root_node() - for edge in node.parents: - if edge.destination_node is root: - return edge.req - return None - def get_all_nodes(self) -> typing.Iterable[DependencyNode]: return self.nodes.values() diff --git a/src/fromager/requirements_file.py b/src/fromager/requirements_file.py index a588d20a3..c7beed399 100644 --- a/src/fromager/requirements_file.py +++ b/src/fromager/requirements_file.py @@ -33,7 +33,6 @@ class SourceType(StrEnum): PREBUILT = "prebuilt" SDIST = "sdist" OVERRIDE = "override" - GIT = "git" def parse_requirements_file( diff --git a/src/fromager/sources.py b/src/fromager/sources.py index cfb600b8f..534573d41 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -67,8 +67,6 @@ def download_url( def get_source_type(ctx: context.WorkContext, req: Requirement) -> SourceType: source_type = SourceType.SDIST - if req.url: - return SourceType.GIT pbi = ctx.package_build_info(req) if ( overrides.find_override_method(req.name, "download_source") @@ -88,27 +86,6 @@ def download_source( download_url: str, ) -> pathlib.Path: logger.info(f"downloading source for {req}") - download_path = pathlib.Path(download_url) - if req.url and download_path.exists(): - logger.info( - "source is already downloaded to %s by cloning %s, ignoring any plugins", - download_url, - req.url, - ) - return download_path - elif req.url: - download_path = ctx.work_dir / f"{req.name}-{version}" / f"{req.name}-{version}" - download_path.mkdir(parents=True, exist_ok=True) - - url_to_clone, git_ref = gitutils.parse_vcs_url(req.url, require_ref=False) - download_git_source( - ctx=ctx, - req=req, - url_to_clone=url_to_clone, - destination_dir=download_path, - ref=git_ref, - ) - return download_path source_path = overrides.find_and_invoke( req.name, @@ -401,43 +378,29 @@ def prepare_source( ) -> pathlib.Path: """Unpack and prepare source for building. - Git URL sources skip the plugin system and are prepared directly. - Non-git sources go through ``find_and_invoke`` which may call a - package-specific override. The plugin may return a ``Path`` or a - ``(Path, bool)`` tuple; both forms are handled. + Calls ``find_and_invoke`` which may call a package-specific override. + The plugin may return a ``Path`` or a ``(Path, bool)`` tuple; both + forms are handled. """ - if req.url: - logger.info( - "preparing source cloned from %s into %s, ignoring any plugins", - req.url, - source_filename, - ) - source_root_dir = pathlib.Path(source_filename) - prepare_new_source( - ctx=ctx, - req=req, - source_root_dir=source_root_dir, - version=version, - ) + logger.info(f"preparing source for {req} from {source_filename}") + prepare_source_details = overrides.find_and_invoke( + req.name, + "prepare_source", + default_prepare_source, + ctx=ctx, + req=req, + source_filename=source_filename, + version=version, + ) + source_root_dir: pathlib.Path + if not isinstance(prepare_source_details, tuple): + source_root_dir = prepare_source_details + elif len(prepare_source_details) == 2: + source_root_dir, _ = prepare_source_details else: - logger.info(f"preparing source for {req} from {source_filename}") - prepare_source_details = overrides.find_and_invoke( - req.name, - "prepare_source", - default_prepare_source, - ctx=ctx, - req=req, - source_filename=source_filename, - version=version, + raise ValueError( + f"do not know how to unpack {prepare_source_details}, expected 1 or 2 members" ) - if not isinstance(prepare_source_details, tuple): - source_root_dir = prepare_source_details - elif len(prepare_source_details) == 2: - source_root_dir, _ = prepare_source_details - else: - raise ValueError( - f"do not know how to unpack {prepare_source_details}, expected 1 or 2 members" - ) write_build_meta(source_root_dir.parent, req, source_filename, version) if source_root_dir is not None: logger.info(f"prepared source for {req} at {source_root_dir}") @@ -524,35 +487,18 @@ def build_sdist( sdist_root_dir=sdist_root_dir, build_env=build_env, ) - if req.url: - # The default approach to making an sdist is to make a tarball from the - # source directory, since most of the time we got the source directory - # by unpacking an existing sdist. When we know we cloned a git repo to - # get the source tree, we can be very sure that creating a tarball will - # NOT produce a valid sdist, so we can use the PEP-517 approach - # instead. - logger.info("using PEP-517 sdist build, ignoring any plugins") - sdist_file = pep517_build_sdist( - ctx=ctx, - extra_environ=extra_environ, - req=req, - sdist_root_dir=sdist_root_dir, - version=version, - build_env=build_env, - ) - else: - sdist_file = overrides.find_and_invoke( - req.name, - "build_sdist", - default_build_sdist, - ctx=ctx, - extra_environ=extra_environ, - req=req, - version=version, - sdist_root_dir=sdist_root_dir, - build_dir=build_dir, - build_env=build_env, - ) + sdist_file: pathlib.Path = overrides.find_and_invoke( + req.name, + "build_sdist", + default_build_sdist, + ctx=ctx, + extra_environ=extra_environ, + req=req, + version=version, + sdist_root_dir=sdist_root_dir, + build_dir=build_dir, + build_env=build_env, + ) logger.info(f"built source distribution {sdist_file}") # validate location and file name diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 0a3c9f7cb..c57ab26f6 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -8,9 +8,8 @@ import pytest from click.testing import CliRunner from packaging.requirements import Requirement -from packaging.version import Version -from fromager import bootstrapper, context, dependency_graph, gitutils, packagesettings +from fromager import context, dependency_graph from fromager.commands import bootstrap @@ -747,190 +746,3 @@ def test_max_release_age_sets_context( assert result.exit_code == 0 assert tmp_context.max_release_age == timedelta(days=45) assert "rejecting versions older than 45 days" in caplog.text - - -@patch("fromager.gitutils.git_clone") -def test_resolve_version_from_git_url_with_submodules_enabled( - mock_git_clone: Mock, - tmp_context: context.WorkContext, -) -> None: - """Test that git_clone is called with submodules=True when configured.""" - req = Requirement("test-pkg @ git+https://github.com/example/repo.git") - - mock_git_options = packagesettings.GitOptions(submodules=True) - - with patch.object(tmp_context, "package_build_info") as mock_pbi: - mock_pbi_instance = Mock() - mock_pbi_instance.git_options = mock_git_options - mock_pbi.return_value = mock_pbi_instance - - with patch( - "fromager.bootstrapper.Bootstrapper._get_version_from_package_metadata" - ) as mock_get_version: - with patch("shutil.move"): - with patch("pathlib.Path.mkdir"): - mock_get_version.return_value = Version("1.0.0") - - # Execute - bs = bootstrapper.Bootstrapper(tmp_context) - try: - bs._resolve_version_from_git_url(req) - except AssertionError: - # Expected since we're mocking everything - pass - - mock_git_clone.assert_called_once() - call_args = mock_git_clone.call_args - assert call_args.kwargs["submodules"] is True - assert call_args.kwargs["repo_url"] == "https://github.com/example/repo.git" - assert call_args.kwargs["ref"] == gitutils.GIT_HEAD - - -@patch("fromager.gitutils.git_clone") -def test_resolve_version_from_git_url_with_specific_submodule_paths( - mock_git_clone: Mock, - tmp_context: context.WorkContext, -) -> None: - """Test that git_clone is called with specific submodule paths when configured.""" - req = Requirement("test-pkg @ git+https://github.com/example/repo.git") - - mock_git_options = packagesettings.GitOptions( - submodule_paths=["vendor/lib1", "vendor/lib2"] - ) - - with patch.object(tmp_context, "package_build_info") as mock_pbi: - mock_pbi_instance = Mock() - mock_pbi_instance.git_options = mock_git_options - mock_pbi.return_value = mock_pbi_instance - - with patch( - "fromager.bootstrapper.Bootstrapper._get_version_from_package_metadata" - ) as mock_get_version: - with patch("shutil.move"): - with patch("pathlib.Path.mkdir"): - mock_get_version.return_value = Version("1.0.0") - - bs = bootstrapper.Bootstrapper(tmp_context) - try: - bs._resolve_version_from_git_url(req) - except AssertionError: - # Expected since we're mocking everything - pass - - mock_git_clone.assert_called_once() - call_args = mock_git_clone.call_args - assert call_args.kwargs["submodules"] == ["vendor/lib1", "vendor/lib2"] - - -@patch("fromager.gitutils.git_clone") -def test_resolve_version_from_git_url_with_submodules_disabled( - mock_git_clone: Mock, - tmp_context: context.WorkContext, -) -> None: - """Test that git_clone is called with submodules=False by default.""" - req = Requirement("test-pkg @ git+https://github.com/example/repo.git") - - with patch( - "fromager.bootstrapper.Bootstrapper._get_version_from_package_metadata" - ) as mock_get_version: - with patch("shutil.move"): - with patch("pathlib.Path.mkdir"): - mock_get_version.return_value = Version("1.0.0") - - bs = bootstrapper.Bootstrapper(tmp_context) - try: - bs._resolve_version_from_git_url(req) - except AssertionError: - # Expected since we're mocking everything - pass - - mock_git_clone.assert_called_once() - call_args = mock_git_clone.call_args - assert call_args.kwargs["submodules"] is False - - -@patch("fromager.gitutils.git_clone") -def test_resolve_version_from_git_url_with_git_ref( - mock_git_clone: Mock, - tmp_context: context.WorkContext, -) -> None: - """Test that git_clone is called with the correct ref when URL includes @ref.""" - req = Requirement("test-pkg @ git+https://github.com/example/repo.git@v1.2.3") - - mock_git_options = packagesettings.GitOptions(submodules=True) - - with patch.object(tmp_context, "package_build_info") as mock_pbi: - mock_pbi_instance = Mock() - mock_pbi_instance.git_options = mock_git_options - mock_pbi.return_value = mock_pbi_instance - - with patch( - "fromager.bootstrapper.Bootstrapper._get_version_from_package_metadata" - ) as mock_get_version: - with patch("shutil.move"): - with patch("pathlib.Path.mkdir"): - mock_get_version.return_value = Version("1.2.3") - - bs = bootstrapper.Bootstrapper(tmp_context) - try: - bs._resolve_version_from_git_url(req) - except AssertionError: - # Expected since we're mocking everything - pass - - mock_git_clone.assert_called_once() - call_args = mock_git_clone.call_args - assert call_args.kwargs["submodules"] is True - assert call_args.kwargs["repo_url"] == "https://github.com/example/repo.git" - assert call_args.kwargs["ref"] == "v1.2.3" - - -def test_resolve_version_from_git_url_invalid_scheme( - tmp_context: context.WorkContext, -) -> None: - """Test that non-git URLs raise ValueError.""" - req = Requirement("test-pkg @ https://github.com/example/repo.git") - - bs = bootstrapper.Bootstrapper(tmp_context) - with pytest.raises(ValueError, match="unsupported VCS URL scheme"): - bs._resolve_version_from_git_url(req) - - -def test_resolve_version_from_git_url_creates_parent_dir_on_first_clone( - tmp_context: context.WorkContext, -) -> None: - """working_src_dir.parent is created before shutil.move on first clone. - - When the URL has no version ref, the version is discovered after cloning, - so working_src_dir is computed inside the clone block. Its parent directory - does not exist yet and must be created before shutil.move is called. - """ - req = Requirement("test-pkg @ git+https://pkg.test/repo.git") - - def fake_download_git_source( - *, - ctx: context.WorkContext, - req: Requirement, - url_to_clone: str, - destination_dir: pathlib.Path, - ref: str, - ) -> None: - destination_dir.mkdir(parents=True, exist_ok=True) - - with ( - patch( - "fromager.sources.download_git_source", - side_effect=fake_download_git_source, - ), - patch( - "fromager.bootstrapper.Bootstrapper._get_version_from_package_metadata", - return_value=Version("1.0.0"), - ), - ): - bs = bootstrapper.Bootstrapper(tmp_context) - src_dir, version = bs._resolve_version_from_git_url(req) - - expected = tmp_context.work_dir / "test-pkg-1.0.0" / "test-pkg-1.0.0" - assert pathlib.Path(src_dir) == expected - assert expected.exists() - assert version == Version("1.0.0") diff --git a/tests/test_bootstrap_requirement_resolver.py b/tests/test_bootstrap_requirement_resolver.py index 4c9148c56..e2a349d84 100644 --- a/tests/test_bootstrap_requirement_resolver.py +++ b/tests/test_bootstrap_requirement_resolver.py @@ -2,7 +2,6 @@ from unittest.mock import MagicMock, patch -import pytest from packaging.requirements import Requirement from packaging.utils import canonicalize_name from packaging.version import Version @@ -433,51 +432,6 @@ def test_resolve_from_graph_name_fallback_returns_none_for_missing_package( assert result is None -def test_resolve_rejects_git_urls_for_source(tmp_context: WorkContext) -> None: - """BootstrapRequirementResolver.resolve() rejects git URLs when pre_built=False.""" - resolver = BootstrapRequirementResolver(tmp_context) - - with pytest.raises( - ValueError, match="Git URL requirements must be handled by Bootstrapper" - ): - resolver.resolve( - req=Requirement("package @ git+https://github.com/example/repo.git"), - req_type=RequirementType.TOP_LEVEL, - pre_built=False, - parent_req=None, - ) - - -@patch("fromager.resolver.find_all_matching_from_provider") -def test_resolve_allows_git_urls_for_prebuilt( - mock_resolve: MagicMock, - tmp_context: WorkContext, -) -> None: - """BootstrapRequirementResolver.resolve() allows git URLs when pre_built=True (test mode fallback).""" - resolver = BootstrapRequirementResolver(tmp_context) - req = Requirement("mypkg @ git+https://github.com/example/repo.git") - - # Mock resolution to return expected result (as list) - mock_resolve.return_value = [ - ("https://files.pythonhosted.org/mypkg-1.0-py3-none-any.whl", Version("1.0")) - ] - - # Should NOT raise - git URLs are allowed when explicitly requesting prebuilt - results = resolver.resolve( - req=req, - req_type=RequirementType.INSTALL, - pre_built=True, - parent_req=None, - ) - - # Verify resolution was called - mock_resolve.assert_called_once() - assert len(results) == 1 - url, version = results[0] - assert url == "https://files.pythonhosted.org/mypkg-1.0-py3-none-any.whl" - assert version == Version("1.0") - - @patch("fromager.resolver.find_all_matching_from_provider") def test_resolve_auto_routes_to_prebuilt( mock_resolve: MagicMock, @@ -562,54 +516,6 @@ def test_resolve_auto_routes_to_source( assert version == Version("2.0") -def test_extend_known_versions_accumulates(tmp_context: WorkContext) -> None: - """extend_known_versions() accumulates versions across calls.""" - resolver = BootstrapRequirementResolver(tmp_context) - req1 = Requirement("mypkg>=1.0") - req2 = Requirement("mypkg>=2.0") - - resolver.extend_known_versions( - req1, - pre_built=False, - result=[("https://files.test/mypkg-1.5.tar.gz", Version("1.5"))], - ) - resolver.extend_known_versions( - req2, - pre_built=False, - result=[("https://files.test/mypkg-2.0.tar.gz", Version("2.0"))], - ) - - # Both versions are now available when filtering by the wider specifier - matching = resolver.get_matching_versions(req1, pre_built=False) - assert len(matching) == 2 - assert matching[0][1] == Version("2.0") - assert matching[1][1] == Version("1.5") - - -def test_get_matching_versions_returns_independent_lists( - tmp_context: WorkContext, -) -> None: - """get_matching_versions() returns a new list each call.""" - resolver = BootstrapRequirementResolver(tmp_context) - req = Requirement("mypkg>=1.0") - - resolver.extend_known_versions( - req, - pre_built=False, - result=[("https://files.test/mypkg-1.0.tar.gz", Version("1.0"))], - ) - list1 = resolver.get_matching_versions(req, pre_built=False) - list2 = resolver.get_matching_versions(req, pre_built=False) - - assert list1 == list2 - assert list1 is not list2 - - # Mutating one does not affect the other - list1.append(("https://files.test/bad.tar.gz", Version("9.9"))) - list3 = resolver.get_matching_versions(req, pre_built=False) - assert len(list3) == 1 - - @patch("fromager.resolver.find_all_matching_from_provider") def test_resolve_cache_returns_independent_lists( mock_resolve: MagicMock, diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 6c662f0da..47c894022 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -1221,3 +1221,13 @@ def test_build_item_do_build_builds_wheel(tmp_context: WorkContext) -> None: ) assert wheel == built_wheel assert sdist == built_sdist + + +def test_resolve_versions_rejects_url_requirement( + tmp_context: WorkContext, +) -> None: + """resolve_versions() raises ValueError for direct-reference URL requirements.""" + req = Requirement("pkg @ git+https://github.com/example/repo.git") + bs = bootstrapper.Bootstrapper(tmp_context) + with pytest.raises(ValueError, match="no longer supported"): + bs.resolve_versions(req=req, req_type=RequirementType.TOP_LEVEL) diff --git a/tests/test_constraints.py b/tests/test_constraints.py index d17485a49..6f1d7dcfd 100644 --- a/tests/test_constraints.py +++ b/tests/test_constraints.py @@ -156,7 +156,7 @@ def test_invalid_constraints() -> None: c.add_constraint("foo") with pytest.raises(InvalidConstraintError, match=r".*has extras"): c.add_constraint("foo[extra]>=1.0") - with pytest.raises(InvalidConstraintError, match=r".*has an url"): + with pytest.raises(InvalidConstraintError, match=r".*has a URL"): c.add_constraint("foo@https://foo.test") diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index f56353827..91838e930 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -886,11 +886,7 @@ def _make_pbi(env_yaml: str, tmp_path: pathlib.Path) -> PackageBuildInfo: def test_version_env_var_raises_when_version_unknown( tmp_path: pathlib.Path, ) -> None: - """Using ${__version__} in env without a fallback raises when version is None. - - This mirrors the git-URL bootstrap path where the version has not yet - been resolved (e.g. ``pkg @ git+https://host/repo.git@main``). - """ + """Using ${__version__} in env without a fallback raises when version is None.""" pbi = _make_pbi( """ env: diff --git a/tests/test_sources.py b/tests/test_sources.py index 38746d02e..1ed0904b8 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -419,15 +419,6 @@ def test_ensure_pkg_info_creates_in_both_dirs( assert (build_dir / "PKG-INFO").is_file() -def test_get_source_type_git( - tmp_context: context.WorkContext, -) -> None: - req = Requirement("pkg @ git+https://github.com/org/pkg.git") - result = sources.get_source_type(tmp_context, req) - - assert result == sources.SourceType.GIT - - @patch("fromager.overrides.find_override_method") def test_get_source_type_override_via_download_source( mock_find: Mock, @@ -599,48 +590,6 @@ def test_unpack_source_reuse_when_no_cleanup( assert (result / "setup.py").read_text() == "# old" -@patch("fromager.sources.download_git_source") -def test_download_source_git_already_cloned( - mock_git: Mock, - tmp_context: context.WorkContext, - tmp_path: pathlib.Path, -) -> None: - clone_dir = tmp_path / "cloned" - clone_dir.mkdir() - req = Requirement("pkg @ git+https://github.com/org/pkg.git") - - result = sources.download_source( - ctx=tmp_context, - req=req, - version=Version("1.0"), - download_url=str(clone_dir), - ) - - assert result == clone_dir - mock_git.assert_not_called() - - -@patch("fromager.sources.download_git_source") -def test_download_source_git_with_ref( - mock_git: Mock, - tmp_context: context.WorkContext, -) -> None: - req = Requirement("pkg @ git+https://github.com/org/pkg.git@v2.0") - - result = sources.download_source( - ctx=tmp_context, - req=req, - version=Version("2.0"), - download_url="/nonexistent", - ) - - mock_git.assert_called_once() - call_kwargs = mock_git.call_args[1] - assert call_kwargs["ref"] == "v2.0" - assert call_kwargs["url_to_clone"] == "https://github.com/org/pkg.git" - assert result == tmp_context.work_dir / "pkg-2.0" / "pkg-2.0" - - @patch("fromager.overrides.find_and_invoke") def test_download_source_regular_package( mock_invoke: Mock,