Merge pull request #607 from LalatenduMohanty/issue_561

Replaceing the metadata parser from packaging.metadata

Author: mergify[bot]

pyproject.toml

@@ -36,7 +36,6 @@ dependencies = [
     "elfdeps>=0.2.0",
     "license-expression",
     "packaging",
-    "pkginfo",
     "psutil",
     "pydantic",
     "pypi_simple",
@@ -204,7 +203,7 @@ exclude = [
 
 [[tool.mypy.overrides]]
 # packages without typing annotations and stubs
-module = ["license_expression", "pyproject_hooks", "requests_mock", "resolver", "stevedore"]
+module = ["hatchling", "hatchling.build", "license_expression", "pyproject_hooks", "requests_mock", "resolver", "stevedore"]
 ignore_missing_imports = true
 
 [tool.basedpyright]

src/fromager/bootstrapper.py

@@ -12,7 +12,6 @@
 import tempfile
 import typing
 import zipfile
-from email.parser import BytesParser
 from urllib.parse import urlparse
 
 from packaging.requirements import Requirement
@@ -1234,10 +1233,10 @@ def _get_version_from_package_metadata(
             config_settings=pbi.config_settings,
         )
         metadata_filename = source_dir.parent / metadata_dir_base / "METADATA"
-        with open(metadata_filename, "rb") as f:
-            p = BytesParser()
-            metadata = p.parse(f, headersonly=True)
-        return Version(metadata["Version"])
+        # Disable validation because some packages have metadata version mismatches
+        # (e.g., declaring Metadata-Version: 2.2 but using fields from 2.4).
+        metadata = dependencies.parse_metadata(metadata_filename, validate=False)
+        return metadata.version
 
     def _create_unpack_dir(
         self, req: Requirement, resolved_version: Version

src/fromager/candidate.py

@@ -2,12 +2,10 @@
 import datetime
 import logging
 import typing
-from email.message import EmailMessage, Message
-from email.parser import BytesParser
 from io import BytesIO
-from typing import TYPE_CHECKING
 from zipfile import ZipFile
 
+from packaging.metadata import Metadata
 from packaging.requirements import Requirement
 from packaging.utils import BuildTag, canonicalize_name
 from packaging.version import Version
@@ -16,13 +14,6 @@
 
 logger = logging.getLogger(__name__)
 
-# fix for runtime errors caused by inheriting classes that are generic in stubs but not runtime
-# https://mypy.readthedocs.io/en/latest/runtime_troubles.html#using-classes-that-are-generic-in-stubs-but-not-at-runtime
-if TYPE_CHECKING:
-    Metadata = Message[str, str]
-else:
-    Metadata = Message
-
 
 @dataclasses.dataclass(frozen=True, order=True, slots=True, repr=False, kw_only=True)
 class Candidate:
@@ -73,11 +64,10 @@ def metadata(self) -> Metadata:
         return self._metadata
 
     def _get_dependencies(self) -> typing.Iterable[Requirement]:
-        deps = self.metadata.get_all("Requires-Dist", [])
+        deps = self.metadata.requires_dist or []
         extras = self.extras if self.extras else [""]
 
-        for d in deps:
-            r = Requirement(d)
+        for r in deps:
             if r.marker is None:
                 yield r
             else:
@@ -95,19 +85,22 @@ def dependencies(self) -> list[Requirement]:
 
     @property
     def requires_python(self) -> str | None:
-        return self.metadata.get("Requires-Python")
+        spec = self.metadata.requires_python
+        return str(spec) if spec is not None else None
 
 
-def get_metadata_for_wheel(url: str, metadata_url: str | None = None) -> Metadata:
-    """
-    Get metadata for a wheel, supporting PEP 658 metadata endpoints.
+def get_metadata_for_wheel(
+    url: str, metadata_url: str | None = None, *, validate: bool = True
+) -> Metadata:
+    """Get metadata for a wheel, supporting PEP 658 metadata endpoints.
 
     Args:
         url: URL of the wheel file
         metadata_url: Optional URL of the metadata file (PEP 658)
+        validate: Whether to validate metadata (default: True)
 
     Returns:
-        Parsed metadata as a Message object
+        Parsed metadata as a Metadata object
     """
     # Try PEP 658 metadata endpoint first if available
     if metadata_url:
@@ -118,9 +111,9 @@ def get_metadata_for_wheel(url: str, metadata_url: str | None = None) -> Metadat
             response = session.get(metadata_url)
             response.raise_for_status()
 
-            # Parse metadata directly from the response content
-            p = BytesParser()
-            metadata = p.parse(BytesIO(response.content), headersonly=True)
+            # Parse metadata directly using packaging.metadata.Metadata
+            # (avoiding circular import with dependencies module)
+            metadata = Metadata.from_email(response.content, validate=validate)
             logger.debug(f"Successfully retrieved metadata via PEP 658 for {url}")
             return metadata
 
@@ -136,8 +129,10 @@ def get_metadata_for_wheel(url: str, metadata_url: str | None = None) -> Metadat
     with ZipFile(BytesIO(data)) as z:
         for n in z.namelist():
             if n.endswith(".dist-info/METADATA"):
-                p = BytesParser()
-                return p.parse(z.open(n), headersonly=True)
+                metadata_content = z.read(n)
+                # Parse metadata directly using packaging.metadata.Metadata
+                # (avoiding circular import with dependencies module)
+                return Metadata.from_email(metadata_content, validate=validate)
 
-    # If we didn't find the metadata, return an empty dict
-    return EmailMessage()
+    # If we didn't find the metadata, raise an error
+    raise ValueError(f"Could not find METADATA file in wheel: {url}")

src/fromager/dependencies.py

@@ -6,13 +6,13 @@
 import pathlib
 import tempfile
 import typing
+import zipfile
 
-import pkginfo
 import pyproject_hooks
 import tomlkit
 from packaging.metadata import Metadata
 from packaging.requirements import Requirement
-from packaging.utils import NormalizedName, canonicalize_name
+from packaging.utils import NormalizedName, canonicalize_name, parse_wheel_filename
 from packaging.version import Version
 
 from . import (
@@ -344,14 +344,23 @@ def default_get_install_dependencies_of_sdist(
     return set(metadata.requires_dist)
 
 
-def parse_metadata(metadata_file: pathlib.Path, *, validate: bool = True) -> Metadata:
-    """Parse a dist-info/METADATA file
+def parse_metadata(
+    metadata_source: pathlib.Path | bytes, *, validate: bool = True
+) -> Metadata:
+    """Parse metadata from a file path or bytes.
+
+    Args:
+        metadata_source: Path to METADATA file or bytes containing metadata
+        validate: Whether to validate metadata (default: True)
 
-    The default parse mode is 'strict'. It even fails for a mismatch of field
-    and core metadata version, e.g. a package with metadata 2.2 and
-    license-expression field (added in 2.4).
+    Returns:
+        Parsed Metadata object
     """
-    return Metadata.from_email(metadata_file.read_bytes(), validate=validate)
+    if isinstance(metadata_source, pathlib.Path):
+        metadata_bytes = metadata_source.read_bytes()
+    else:
+        metadata_bytes = metadata_source
+    return Metadata.from_email(metadata_bytes, validate=validate)
 
 
 def pep517_metadata_of_sdist(
@@ -418,16 +427,65 @@ def validate_dist_name_version(
 def get_install_dependencies_of_wheel(
     req: Requirement, wheel_filename: pathlib.Path, requirements_file_dir: pathlib.Path
 ) -> set[Requirement]:
+    """Get install dependencies from a wheel file.
+
+    Extracts and parses the METADATA file from the wheel to get the
+    Requires-Dist entries.
+
+    Args:
+        req: The requirement being processed
+        wheel_filename: Path to the wheel file
+        requirements_file_dir: Directory to write the requirements file
+
+    Returns:
+        Set of requirements from the wheel's metadata
+    """
     logger.info(f"getting installation dependencies from {wheel_filename}")
-    wheel = pkginfo.Wheel(str(wheel_filename))
-    deps = _filter_requirements(req, wheel.requires_dist)
+    # Disable validation because many third-party packages have metadata version
+    # mismatches (e.g., setuptools declares Metadata-Version: 2.2 but uses
+    # license-file which was introduced in 2.4). The old pkginfo library
+    # didn't validate this, so we maintain backward compatibility.
+    metadata = _get_metadata_from_wheel(wheel_filename, validate=False)
+    requires_dist = metadata.requires_dist or []
+    deps = _filter_requirements(req, requires_dist)
     _write_requirements_file(
         deps,
         requirements_file_dir / INSTALL_REQ_FILE_NAME,
     )
     return deps
 
 
+def _get_metadata_from_wheel(
+    wheel_filename: pathlib.Path, *, validate: bool = True
+) -> Metadata:
+    """Extract and parse METADATA from a wheel file.
+
+    Args:
+        wheel_filename: Path to the wheel file
+        validate: Whether to validate metadata (default: True)
+
+    Returns:
+        Parsed Metadata object
+
+    Raises:
+        ValueError: If no METADATA file is found in the wheel
+    """
+    # Get dist-info path from wheel filename.
+    # Uses same pattern as wheels.extract_info_from_wheel_file:
+    _, dist_version, _, _ = parse_wheel_filename(wheel_filename.name)
+    dist_name = wheel_filename.name.split("-", 1)[0]
+    metadata_path = f"{dist_name}-{dist_version}.dist-info/METADATA"
+
+    with zipfile.ZipFile(wheel_filename) as whl:
+        try:
+            metadata_content = whl.read(metadata_path)
+        except KeyError:
+            raise ValueError(
+                f"Could not find METADATA file in wheel: {wheel_filename}"
+            ) from None
+        return parse_metadata(metadata_content, validate=validate)
+
+
 def get_pyproject_contents(sdist_root_dir: pathlib.Path) -> dict[str, typing.Any]:
     pyproject_toml_filename = sdist_root_dir / "pyproject.toml"
     if not os.path.exists(pyproject_toml_filename):