Merge pull request #869 from tiran/git-clone-fast

feat: fast, simple git clone

Author: mergify[bot]

src/fromager/gitutils.py

@@ -61,3 +61,76 @@ def git_clone(
         )
 
     return output_dir
+
+
+def git_clone_fast(
+    *,
+    output_dir: pathlib.Path,
+    repo_url: str,
+    ref: str = "HEAD",
+) -> None:
+    """Efficient, blobless git clone with all submodules
+
+    The function clones a git repository with submodules with a blob filter.
+    A blobless clone contains the full git history but no blobs. Blobs are
+    downloaded on demand. Pip's VCS feature uses similar tricks to speed
+    up builds from a VCS URL.
+
+    The *ref* parameter can be any tree-ish reference like a commit, tag, or
+    branch. To force a tag, use ``refs/tags/v1.0`` to fetch the ``v1.0`` tag.
+
+    Like in :command:`pip`, submodules are automatically cloned recursively.
+
+    .. note::
+
+       :command:`git` and ``libcurl`` do not support :envvar:`NETRC`. Use
+       :file:`~/.netrc` or :file:`.gitconfig` for authentication.
+    """
+    parsed_url = urlparse(repo_url)
+    # Create a clean URL without any credentials for logging
+    clean_url = parsed_url._replace(netloc=parsed_url.hostname or "").geturl()
+    logger.info(
+        "cloning %s, tree-ish %r, into %s",
+        clean_url,
+        ref,
+        output_dir,
+    )
+
+    # Clone repo without blobs, don't check out HEAD
+    cmd: list[str]
+    cmd = [
+        "git",
+        "clone",
+        "--filter=blob:none",
+        "--no-checkout",
+        repo_url,
+        str(output_dir),
+    ]
+    external_commands.run(cmd, network_isolation=False)
+
+    # check out reference / tag
+    logger.debug("check out ref")
+    cmd = [
+        "git",
+        "checkout",
+        ref,
+    ]
+    external_commands.run(cmd, cwd=str(output_dir), network_isolation=False)
+
+    # clone submodules if ".gitmodules" exist.
+    if output_dir.joinpath(".gitmodules").is_file():
+        # recursive clone of all submodules, filter out unnecessary blobs,
+        # 4 jobs in parallel
+        logger.debug("update submodules")
+        cmd = [
+            "git",
+            "submodule",
+            "update",
+            "--init",
+            "--recursive",
+            "--filter=blob:none",
+            "--jobs=4",
+        ]
+        external_commands.run(cmd, cwd=str(output_dir), network_isolation=False)
+    else:
+        logger.debug("no .gitmodules file")

tests/test_gitutils.py

@@ -0,0 +1,63 @@
+import pathlib
+from unittest.mock import Mock, patch
+
+import pytest
+
+from fromager.gitutils import git_clone_fast
+
+
+@patch("fromager.external_commands.run")
+def test_git_clone_fast(m_run: Mock, tmp_path: pathlib.Path) -> None:
+    repo_url = "https://git.test/project.git"
+    git_clone_fast(output_dir=tmp_path, repo_url=repo_url)
+
+    assert m_run.call_count == 2
+    m_run.assert_any_call(
+        [
+            "git",
+            "clone",
+            "--filter=blob:none",
+            "--no-checkout",
+            repo_url,
+            str(tmp_path),
+        ],
+        network_isolation=False,
+    )
+    m_run.assert_any_call(
+        [
+            "git",
+            "checkout",
+            "HEAD",
+        ],
+        network_isolation=False,
+        cwd=str(tmp_path),
+    )
+
+
+@patch("fromager.external_commands.run")
+def test_git_clone_fast_submodules(m_run: Mock, tmp_path: pathlib.Path) -> None:
+    repo_url = "https://git.test/project.git"
+    tmp_path.joinpath(".gitmodules").touch()
+    git_clone_fast(output_dir=tmp_path, repo_url=repo_url)
+
+    assert m_run.call_count == 3
+    m_run.assert_called_with(
+        [
+            "git",
+            "submodule",
+            "update",
+            "--init",
+            "--recursive",
+            "--filter=blob:none",
+            "--jobs=4",
+        ],
+        cwd=str(tmp_path),
+        network_isolation=False,
+    )
+
+
+@pytest.mark.skip(reason="needs network access")
+def test_git_clone_real(tmp_path: pathlib.Path) -> None:
+    repo_url = "https://github.com/python-wheel-build/fromager.git"
+    git_clone_fast(output_dir=tmp_path, repo_url=repo_url, ref="refs/tags/0.73.0")
+    assert tmp_path.joinpath("src", "fromager").is_dir()