From 230c3632a09d23713dfef987d5af5eeeee86b5ec Mon Sep 17 00:00:00 2001 From: Pacheco <41350439+joaogabriel15@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:07:39 -0300 Subject: [PATCH 1/2] Stream package extraction to bound peak memory usage --- src/manage/install_command.py | 5 +++-- tests/test_install_command.py | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/manage/install_command.py b/src/manage/install_command.py index bcfe9d8..5e3278a 100644 --- a/src/manage/install_command.py +++ b/src/manage/install_command.py @@ -166,6 +166,7 @@ def validate_package(install, dest, *, delete=True): def extract_package(package, prefix, calculate_dest=Path, *, on_progress=None, repair=False): + import shutil import zipfile LOGGER.debug("Starting extract of %s to %s", package, prefix) @@ -205,8 +206,8 @@ def _calc(prefix, filename, calculate_dest=calculate_dest): warn_overwrite.append(dest) continue ensure_tree(dest) - with open(dest, "wb") as f: - f.write(zf.read(member)) + with zf.open(member) as source, open(dest, "wb") as f: + shutil.copyfileobj(source, f, length=1024 * 1024) on_progress(100) if warn_out_of_prefix: diff --git a/tests/test_install_command.py b/tests/test_install_command.py index 7e8991d..6cb6d1b 100644 --- a/tests/test_install_command.py +++ b/tests/test_install_command.py @@ -10,6 +10,48 @@ from manage.logging import LOGGER +@pytest.mark.parametrize("suffix", [".zip", ".nupkg"]) +@pytest.mark.parametrize("repair", [False, True]) +def test_extract_package_streaming(tmp_path, monkeypatch, suffix, repair): + """Extract in bounded reads while preserving overwrite and repair behavior.""" + import zipfile + + package = tmp_path / ("package" + suffix) + prefix = tmp_path / "install" + prefix.mkdir() + existing = prefix / "existing.txt" + existing.write_bytes(b"original") + data = bytes(range(256)) * 10000 + archive_prefix = "tools/" if suffix == ".nupkg" else "" + with zipfile.ZipFile(package, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr(archive_prefix + "nested/data.bin", data) + zf.writestr(archive_prefix + "empty.txt", b"") + zf.writestr(archive_prefix + "existing.txt", b"replacement") + if suffix == ".nupkg": + zf.writestr("metadata.txt", b"ignored") + + reads = [] + original_read = zipfile.ZipExtFile.read + + def bounded_read(self, n=-1): + assert 0 < n <= 1024 * 1024 + reads.append(n) + return original_read(self, n) + + monkeypatch.setattr(zipfile.ZipExtFile, "read", bounded_read) + progress = [] + IC.extract_package(package, prefix, calculate_dest=Path, + on_progress=progress.append, repair=repair) + assert (prefix / "nested/data.bin").read_bytes() == data + assert (prefix / "empty.txt").read_bytes() == b"" + assert existing.read_bytes() == (b"replacement" if repair else b"original") + assert not (prefix / "metadata.txt").exists() + assert len(reads) >= 4 + assert progress[0] == 0 + assert 100 in progress + assert (None in progress) == (not repair) + + def test_print_cli_shortcuts(patched_installs, assert_log, monkeypatch, tmp_path): class Cmd: scratch = {} From 44a0114f52c139a249f17d4d0de5e890321ed0ac Mon Sep 17 00:00:00 2001 From: Pacheco <41350439+joaogabriel15@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:34:18 -0300 Subject: [PATCH 2/2] Use 10 MiB chunks for package extraction --- src/manage/install_command.py | 2 +- tests/test_install_command.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/manage/install_command.py b/src/manage/install_command.py index 5e3278a..89b589e 100644 --- a/src/manage/install_command.py +++ b/src/manage/install_command.py @@ -207,7 +207,7 @@ def _calc(prefix, filename, calculate_dest=calculate_dest): continue ensure_tree(dest) with zf.open(member) as source, open(dest, "wb") as f: - shutil.copyfileobj(source, f, length=1024 * 1024) + shutil.copyfileobj(source, f, length=10 * 1024 * 1024) on_progress(100) if warn_out_of_prefix: diff --git a/tests/test_install_command.py b/tests/test_install_command.py index 6cb6d1b..45b6cdd 100644 --- a/tests/test_install_command.py +++ b/tests/test_install_command.py @@ -21,7 +21,8 @@ def test_extract_package_streaming(tmp_path, monkeypatch, suffix, repair): prefix.mkdir() existing = prefix / "existing.txt" existing.write_bytes(b"original") - data = bytes(range(256)) * 10000 + chunk_size = 10 * 1024 * 1024 + data = bytes(range(256)) * (chunk_size // 256 + 1) archive_prefix = "tools/" if suffix == ".nupkg" else "" with zipfile.ZipFile(package, "w", zipfile.ZIP_DEFLATED) as zf: zf.writestr(archive_prefix + "nested/data.bin", data) @@ -34,9 +35,11 @@ def test_extract_package_streaming(tmp_path, monkeypatch, suffix, repair): original_read = zipfile.ZipExtFile.read def bounded_read(self, n=-1): - assert 0 < n <= 1024 * 1024 - reads.append(n) - return original_read(self, n) + assert n == chunk_size + result = original_read(self, n) + if self.name == archive_prefix + "nested/data.bin": + reads.append(len(result)) + return result monkeypatch.setattr(zipfile.ZipExtFile, "read", bounded_read) progress = [] @@ -46,7 +49,7 @@ def bounded_read(self, n=-1): assert (prefix / "empty.txt").read_bytes() == b"" assert existing.read_bytes() == (b"replacement" if repair else b"original") assert not (prefix / "metadata.txt").exists() - assert len(reads) >= 4 + assert reads == [chunk_size, 256, 0] assert progress[0] == 0 assert 100 in progress assert (None in progress) == (not repair)