feat: speed up modpack processing with multithreading and session management

Author: devsker

server_pack_builder.py

@@ -6,8 +6,10 @@
 import shutil
 import sys
 import tempfile
+import threading
 import zipfile
-from typing import Optional, Tuple
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Dict, Optional, Tuple
 from urllib.parse import urlparse
 
 import requests
@@ -32,6 +34,22 @@ def setup_logging(verbose: bool):
     )
 
 
+def get_default_worker_count() -> int:
+    cpu_count = os.cpu_count() or 1
+    return max(4, min(32, cpu_count * 4))
+
+
+_thread_local = threading.local()
+
+
+def get_requests_session() -> requests.Session:
+    session = getattr(_thread_local, "session", None)
+    if session is None:
+        session = requests.Session()
+        _thread_local.session = session
+    return session
+
+
 def is_fabric_client_only(jar_path: str) -> Tuple[Optional[bool], str]:
     """
     Checks if a jar is a Fabric mod and if it is client-side only.
@@ -148,13 +166,27 @@ def process_local_modpack(source: str, dest: str, dry_run: bool):
 
     logging.info(f"Scanning {total_mods} JAR files in '{source}'...")
 
-    for filename in files:
+    def analyze_jar(index: int, filename: str) -> Tuple[int, str, str, bool, str, str]:
         file_path = os.path.join(source, filename)
-        processed += 1
-
         is_client, mod_type, reason = check_jar_sidedness(file_path)
+        return index, filename, file_path, is_client, mod_type, reason
+
+    max_workers = get_default_worker_count()
+    results: Dict[int, Tuple[str, str, bool, str, str]] = {}
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [
+            executor.submit(analyze_jar, index, filename)
+            for index, filename in enumerate(files)
+        ]
+
+        for future in as_completed(futures):
+            index, filename, file_path, is_client, mod_type, reason = future.result()
+            results[index] = (filename, file_path, is_client, mod_type, reason)
+
+    for index in range(len(files)):
+        filename, file_path, is_client, mod_type, reason = results[index]
+        processed += 1
 
-        # Action based on sidedness
         if is_client:
             logging.info(f"[SKIP] {filename} ({mod_type}): {reason}")
             skipped += 1
@@ -210,7 +242,8 @@ def process_modrinth_pack(modrinth_url: str, output_file: Optional[str], dry_run
 
     try:
         # Get Project Info
-        project_resp = requests.get(f"https://api.modrinth.com/v2/project/{slug}")
+        session = get_requests_session()
+        project_resp = session.get(f"https://api.modrinth.com/v2/project/{slug}")
         if project_resp.status_code != 200:
             logging.error(
                 f"Failed to fetch project info: {project_resp.status_code} {project_resp.text}"
@@ -228,14 +261,10 @@ def process_modrinth_pack(modrinth_url: str, output_file: Optional[str], dry_run
 
         # Get Version Info
         if version_id:
-            version_resp = requests.get(
-                f"https://api.modrinth.com/v2/version/{version_id}"
-            )
+            version_resp = session.get(f"https://api.modrinth.com/v2/version/{version_id}")
         else:
             # Get versions list and pick latest
-            version_resp = requests.get(
-                f"https://api.modrinth.com/v2/project/{slug}/version"
-            )
+            version_resp = session.get(f"https://api.modrinth.com/v2/project/{slug}/version")
 
         if version_resp.status_code != 200:
             logging.error(
@@ -276,10 +305,10 @@ def process_modrinth_pack(modrinth_url: str, output_file: Optional[str], dry_run
 
             logging.info(f"Downloading modpack from {mrpack_url}...")
             if not dry_run:
-                with requests.get(mrpack_url, stream=True) as r:
+                with session.get(mrpack_url, stream=True) as r:
                     r.raise_for_status()
                     with open(pack_path, "wb") as f:
-                        for chunk in r.iter_content(chunk_size=8192):
+                        for chunk in r.iter_content(chunk_size=65536):
                             f.write(chunk)
             else:
                 # Mock file for dry run if checking structure logic (but we need real file to extract index)
@@ -289,10 +318,10 @@ def process_modrinth_pack(modrinth_url: str, output_file: Optional[str], dry_run
                 # Dry run usually means "don't produce final output/change system state".
                 # We will download to temp, analyze, but not write final output.
                 logging.info("(Dry Run) Downloading for analysis...")
-                with requests.get(mrpack_url, stream=True) as r:
+                with session.get(mrpack_url, stream=True) as r:
                     r.raise_for_status()
                     with open(pack_path, "wb") as f:
-                        for chunk in r.iter_content(chunk_size=8192):
+                        for chunk in r.iter_content(chunk_size=65536):
                             f.write(chunk)
 
             # Extract modrinth.index.json
@@ -330,43 +359,72 @@ def process_modrinth_pack(modrinth_url: str, output_file: Optional[str], dry_run
             kept = 0
             skipped = 0
 
-            for mod_entry in files_list:
-                processed += 1
+            def check_mod_entry(
+                index: int, mod_entry: dict
+            ) -> Tuple[int, dict, bool, str, str, str]:
                 path = mod_entry.get("path", "")
-
-                # Only check mods (usually in mods/ directory)
-                # Some packs might put things elsewhere, but standard is mods/
-                if not path.startswith("mods/"):
-                    # Keep non-mod files (resource packs etc might be here? usually they are, but safe to keep?)
-                    # If it's a resource pack, it's client only usually.
-                    # Safe bet: If it ends in .jar, check it. If not, keep it?
-                    # Let's assume everything in 'files' is a mod or resource.
-                    # We will download everything to check.
-                    pass
-
-                download_url = mod_entry["downloads"][0]  # Primary download
                 filename = os.path.basename(path)
-                temp_jar_path = os.path.join(temp_dir, filename)
+                download_url = mod_entry["downloads"][0]
 
-                # Download mod JAR
                 logging.debug(f"Checking {filename}...")
+                temp_handle, temp_jar_path = tempfile.mkstemp(
+                    suffix=".jar", dir=temp_dir
+                )
+                os.close(temp_handle)
+
                 try:
-                    with requests.get(download_url, stream=True) as r:
+                    session = get_requests_session()
+                    with session.get(download_url, stream=True) as r:
                         r.raise_for_status()
                         with open(temp_jar_path, "wb") as f:
-                            for chunk in r.iter_content(chunk_size=8192):
+                            for chunk in r.iter_content(chunk_size=65536):
                                 f.write(chunk)
                 except Exception as e:
-                    logging.warning(
-                        f"Failed to download {filename}: {e}. Skipping check, assuming kept."
+                    try:
+                        os.remove(temp_jar_path)
+                    except OSError:
+                        pass
+                    return (
+                        index,
+                        mod_entry,
+                        False,
+                        "Unknown",
+                        f"Failed to download: {e}",
+                        filename,
                     )
-                    new_files_list.append(mod_entry)
-                    kept += 1
-                    continue
 
-                # Check sidedness
                 is_client, mod_type, reason = check_jar_sidedness(temp_jar_path)
 
+                try:
+                    os.remove(temp_jar_path)
+                except OSError:
+                    pass
+
+                return index, mod_entry, is_client, mod_type, reason, filename
+
+            max_workers = get_default_worker_count()
+            results: Dict[int, Tuple[dict, bool, str, str, str]] = {}
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                futures = [
+                    executor.submit(check_mod_entry, index, mod_entry)
+                    for index, mod_entry in enumerate(files_list)
+                ]
+
+                for future in as_completed(futures):
+                    (
+                        index,
+                        mod_entry,
+                        is_client,
+                        mod_type,
+                        reason,
+                        filename,
+                    ) = future.result()
+                    results[index] = (mod_entry, is_client, mod_type, reason, filename)
+
+            for index in range(len(files_list)):
+                processed += 1
+                mod_entry, is_client, mod_type, reason, filename = results[index]
+
                 if is_client:
                     logging.info(f"[SKIP] {filename} ({mod_type}): {reason}")
                     skipped += 1
@@ -375,12 +433,6 @@ def process_modrinth_pack(modrinth_url: str, output_file: Optional[str], dry_run
                     new_files_list.append(mod_entry)
                     kept += 1
 
-                # Remove temp jar to save space
-                try:
-                    os.remove(temp_jar_path)
-                except OSError:
-                    pass
-
             # Write new index
             index_data["files"] = new_files_list
             index_data["name"] = f"{index_data.get('name', project_title)} (Server)"