Skip to content

Commit a9ef9dd

Browse files
mergify[bot]mergify[bot]
authored
Merge pull request #842 from iangelak/761-fix-bootstrap-parallel-git-urls
fix: git URLs working with bootstrap-parallel
2 parents c2604a0 + 5687cf7 commit a9ef9dd

8 files changed

Lines changed: 190 additions & 3 deletions

File tree

‎.github/workflows/test.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ jobs:
8686
- bootstrap_git_url
8787
- bootstrap_git_url_tag
8888
- bootstrap_parallel
89+
- bootstrap_parallel_git_url
90+
- bootstrap_parallel_git_url_tag
8991
- bootstrap_skip_constraints
9092
- build
9193
- build_order

‎.mergify.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ pull_request_rules:
4646
- check-success=e2e (3.11, 1.75, bootstrap_git_url, ubuntu-latest)
4747
- check-success=e2e (3.11, 1.75, bootstrap_git_url_tag, ubuntu-latest)
4848
- check-success=e2e (3.11, 1.75, bootstrap_parallel, ubuntu-latest)
49+
- check-success=e2e (3.11, 1.75, bootstrap_parallel_git_url, ubuntu-latest)
50+
- check-success=e2e (3.11, 1.75, bootstrap_parallel_git_url_tag, ubuntu-latest)
4951
- check-success=e2e (3.11, 1.75, bootstrap_prerelease, ubuntu-latest)
5052
- check-success=e2e (3.11, 1.75, bootstrap_sdist_only, ubuntu-latest)
5153
- check-success=e2e (3.11, 1.75, bootstrap_skip_constraints, ubuntu-latest)
@@ -88,6 +90,10 @@ pull_request_rules:
8890
- check-success=e2e (3.12, 1.75, bootstrap_git_url_tag, ubuntu-latest)
8991
- check-success=e2e (3.12, 1.75, bootstrap_parallel, macos-latest)
9092
- check-success=e2e (3.12, 1.75, bootstrap_parallel, ubuntu-latest)
93+
- check-success=e2e (3.12, 1.75, bootstrap_parallel_git_url, macos-latest)
94+
- check-success=e2e (3.12, 1.75, bootstrap_parallel_git_url, ubuntu-latest)
95+
- check-success=e2e (3.12, 1.75, bootstrap_parallel_git_url_tag, macos-latest)
96+
- check-success=e2e (3.12, 1.75, bootstrap_parallel_git_url_tag, ubuntu-latest)
9197
- check-success=e2e (3.12, 1.75, bootstrap_prerelease, macos-latest)
9298
- check-success=e2e (3.12, 1.75, bootstrap_prerelease, ubuntu-latest)
9399
- check-success=e2e (3.12, 1.75, bootstrap_sdist_only, macos-latest)

‎docs/how-tos/build-from-git-repo.rst‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,15 @@ you can do the following:
1414
This will clone the ``stevedore`` repository and build the package from the
1515
local copy.
1616

17+
You can also use the ``bootstrap-parallel`` command for faster builds:
18+
19+
.. code-block:: console
20+
21+
$ fromager bootstrap-parallel stevedore @ git+https://github.com/openstack/stevedore.git
22+
23+
This will perform the same operation but build wheels in parallel after the
24+
bootstrap phase completes.
25+
1726
Building from a specific version
1827
--------------------------------
1928

@@ -24,9 +33,15 @@ the ``@`` syntax to specify the version.
2433
2534
$ fromager bootstrap stevedore @ git+https://github.com/openstack/stevedore.git@5.2.0
2635
27-
This will clone the ``stevedore`` repository at the tagg ``5.2.0`` and build the
36+
This will clone the ``stevedore`` repository at the tag ``5.2.0`` and build the
2837
package from the local copy.
2938

39+
Or with parallel builds:
40+
41+
.. code-block:: console
42+
43+
$ fromager bootstrap-parallel stevedore @ git+https://github.com/openstack/stevedore.git@5.2.0
44+
3045
.. important::
3146

3247
Building from a git repository URL is a special case which bypasses all of

‎e2e/test_bootstrap_parallel_git_url.sh‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
3+
# Test bootstrapping from a requirement with a git+https URL witout specifying a
4+
# version tag.
5+
6+
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
7+
source "$SCRIPTDIR/common.sh"
8+
9+
GIT_REPO_URL="https://github.com/python-wheel-build/stevedore-test-repo.git"
10+
11+
fromager \
12+
--debug \
13+
--log-file="$OUTDIR/bootstrap.log" \
14+
--error-log-file="$OUTDIR/fromager-errors.log" \
15+
--sdists-repo="$OUTDIR/sdists-repo" \
16+
--wheels-repo="$OUTDIR/wheels-repo" \
17+
--work-dir="$OUTDIR/work-dir" \
18+
--settings-dir="$SCRIPTDIR/changelog_settings" \
19+
bootstrap-parallel "stevedore @ git+${GIT_REPO_URL}"
20+
21+
find "$OUTDIR/wheels-repo/" -name '*.whl'
22+
find "$OUTDIR/sdists-repo/" -name '*.tar.gz'
23+
ls "$OUTDIR"/work-dir/*/build.log || true
24+
25+
EXPECTED_FILES="
26+
$OUTDIR/wheels-repo/downloads/setuptools-*.whl
27+
$OUTDIR/wheels-repo/downloads/pbr-*.whl
28+
$OUTDIR/wheels-repo/downloads/stevedore-*.whl
29+
30+
$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
31+
$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
32+
33+
$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz
34+
$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz
35+
$OUTDIR/sdists-repo/builds/pbr-*.tar.gz
36+
37+
$OUTDIR/work-dir/build-order.json
38+
$OUTDIR/work-dir/constraints.txt
39+
40+
$OUTDIR/bootstrap.log
41+
$OUTDIR/fromager-errors.log
42+
43+
$OUTDIR/work-dir/pbr-*/build.log
44+
$OUTDIR/work-dir/setuptools-*/build.log
45+
$OUTDIR/work-dir/stevedore-*/build.log
46+
"
47+
48+
pass=true
49+
for pattern in $EXPECTED_FILES; do
50+
if [ ! -f "${pattern}" ]; then
51+
echo "Did not find $pattern" 1>&2
52+
pass=false
53+
fi
54+
done
55+
56+
$pass
57+
58+
twine check "$OUTDIR"/sdists-repo/builds/*.tar.gz
59+
twine check "$OUTDIR"/wheels-repo/downloads/*.whl
60+

‎e2e/test_bootstrap_parallel_git_url_tag.sh‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
3+
# Test bootstrapping from a requirement with a git+https URL and specifying a
4+
# version tag.
5+
6+
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
7+
source "$SCRIPTDIR/common.sh"
8+
9+
GIT_REPO_URL="https://github.com/python-wheel-build/stevedore-test-repo.git"
10+
11+
fromager \
12+
--debug \
13+
--log-file="$OUTDIR/bootstrap.log" \
14+
--error-log-file="$OUTDIR/fromager-errors.log" \
15+
--sdists-repo="$OUTDIR/sdists-repo" \
16+
--wheels-repo="$OUTDIR/wheels-repo" \
17+
--work-dir="$OUTDIR/work-dir" \
18+
--settings-dir="$SCRIPTDIR/changelog_settings" \
19+
bootstrap-parallel "stevedore @ git+${GIT_REPO_URL}@5.2.0"
20+
21+
find "$OUTDIR/wheels-repo/" -name '*.whl'
22+
find "$OUTDIR/sdists-repo/" -name '*.tar.gz'
23+
ls "$OUTDIR"/work-dir/*/build.log || true
24+
25+
EXPECTED_FILES="
26+
$OUTDIR/wheels-repo/downloads/setuptools-*.whl
27+
$OUTDIR/wheels-repo/downloads/pbr-*.whl
28+
$OUTDIR/wheels-repo/downloads/stevedore-*.whl
29+
30+
$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
31+
$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
32+
33+
$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz
34+
$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz
35+
$OUTDIR/sdists-repo/builds/pbr-*.tar.gz
36+
37+
$OUTDIR/work-dir/build-order.json
38+
$OUTDIR/work-dir/constraints.txt
39+
40+
$OUTDIR/bootstrap.log
41+
$OUTDIR/fromager-errors.log
42+
43+
$OUTDIR/work-dir/pbr-*/build.log
44+
$OUTDIR/work-dir/setuptools-*/build.log
45+
$OUTDIR/work-dir/stevedore-*/build.log
46+
"
47+
48+
pass=true
49+
for pattern in $EXPECTED_FILES; do
50+
if [ ! -f "${pattern}" ]; then
51+
echo "Did not find $pattern" 1>&2
52+
pass=false
53+
fi
54+
done
55+
56+
$pass
57+
58+
twine check "$OUTDIR"/sdists-repo/builds/*.tar.gz
59+
twine check "$OUTDIR"/wheels-repo/downloads/*.whl
60+

‎src/fromager/commands/build.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -661,11 +661,20 @@ def update_progressbar_cb(future: concurrent.futures.Future) -> None:
661661
logger.info("requires exclusive build")
662662
logger.info("ready to build")
663663

664+
# Use original top-level requirement only if it has a URL (git, etc.),
665+
# otherwise use the resolved version to avoid building wrong versions.
666+
top_level_req = graph.get_top_level_requirement(node)
667+
if top_level_req and top_level_req.url:
668+
req = top_level_req
669+
logger.debug("using top-level requirement with URL: %s", req)
670+
else:
671+
req = Requirement(f"{node.canonicalized_name}=={node.version}")
672+
664673
future = executor.submit(
665674
_build_parallel,
666675
wkctx=wkctx,
667676
resolved_version=node.version,
668-
req=node.requirement,
677+
req=req,
669678
source_download_url=node.download_url,
670679
force=force,
671680
cache_wheel_server_url=cache_wheel_server_url,

‎src/fromager/dependency_graph.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,19 @@ def get_nodes_by_name(self, req_name: str | None) -> list[DependencyNode]:
367367
def get_root_node(self) -> DependencyNode:
368368
return self.nodes[ROOT]
369369

370+
def get_top_level_requirement(self, node: DependencyNode) -> Requirement | None:
371+
"""Get the top-level requirement specification for a node.
372+
373+
For packages that were specified as top-level requirements (e.g., with git URLs),
374+
this returns the original requirement specification. Returns None if the node
375+
is not a direct child of ROOT.
376+
"""
377+
root = self.get_root_node()
378+
for edge in node.parents:
379+
if edge.destination_node is root:
380+
return edge.req
381+
return None
382+
370383
def get_all_nodes(self) -> typing.Iterable[DependencyNode]:
371384
return self.nodes.values()
372385

‎src/fromager/sources.py‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import tarfile
99
import typing
1010
import zipfile
11+
from urllib.parse import urlparse
1112

1213
import resolvelib
1314
from packaging.requirements import Requirement
@@ -76,11 +77,32 @@ def download_source(
7677
elif req.url:
7778
download_path = ctx.work_dir / f"{req.name}-{version}" / f"{req.name}-{version}"
7879
download_path.mkdir(parents=True, exist_ok=True)
80+
81+
# Parse ref from URL if it's in the form repo@ref
82+
url_to_clone = req.url
83+
git_ref: str | None = None
84+
85+
# Remove git+ prefix for parsing
86+
parsing_url = url_to_clone
87+
if parsing_url.startswith("git+"):
88+
parsing_url = parsing_url[len("git+") :]
89+
90+
parsed_url = urlparse(parsing_url)
91+
if "@" in parsed_url.path:
92+
# Extract the ref from the path (e.g., /path/to/repo.git@5.2.0 -> @5.2.0)
93+
new_path, _, git_ref = parsed_url.path.rpartition("@")
94+
# Reconstruct URL without the @ref part
95+
url_to_clone = parsed_url._replace(path=new_path).geturl()
96+
# Add back git+ prefix if it was present
97+
if req.url.startswith("git+"):
98+
url_to_clone = f"git+{url_to_clone}"
99+
79100
download_git_source(
80101
ctx=ctx,
81102
req=req,
82-
url_to_clone=req.url,
103+
url_to_clone=url_to_clone,
83104
destination_dir=download_path,
105+
ref=git_ref,
84106
)
85107
return download_path
86108

0 commit comments

Comments
 (0)