Add PackageSpec.from_wheel

.github/workflows/main.yml

@@ -24,6 +24,9 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -e ".[dev]"
+    - name: Build
+      run: |
+        pyproject-build
     - name: Test with pytest
       run: |
         pytest --cov=. --cov-report=xml

CHANGELOG.md

@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Adds `PackageSpec.from_wheel` for generating a package spec from a `.whl` file
+  [#18](https://github.com/pyodide/pyodide-lock/pull/18)
 - Adds `parse_top_level_import_name` for finding importable names in `.whl` files
   [#17](https://github.com/pyodide/pyodide-lock/pull/17)
 

pyodide_lock/__init__.py

@@ -1,7 +1,8 @@
-from .spec import PyodideLockSpec
+from .spec import PackageSpec, PyodideLockSpec
 from .utils import parse_top_level_import_name
 
 __all__ = [
     "PyodideLockSpec",
+    "PackageSpec",
     "parse_top_level_import_name",
 ]

pyodide_lock/spec.py

@@ -4,7 +4,11 @@
 
 from pydantic import BaseModel, ConfigDict
 
-from .utils import _generate_package_hash
+from .utils import (
+    _generate_package_hash,
+    _wheel_depends,
+    parse_top_level_import_name,
+)
 
 
 class InfoSpec(BaseModel):
@@ -33,6 +37,36 @@ class PackageSpec(BaseModel):
     # This field is deprecated
     shared_library: bool = False
 
+    @classmethod
+    def from_wheel(
+        cls,
+        path: Path,
+        marker_env: None | dict[str, str] = None,
+    ) -> "PackageSpec":
+        """Build a package spec from an on-disk wheel.
+
+        This currently assumes a "simple" noarch wheel: more complex packages
+        may require further postprocessing.
+        """
+        import pkginfo
+        from packaging.utils import canonicalize_name
+
+        metadata = pkginfo.get_metadata(str(path))
+
+        if not metadata:
+            raise RuntimeError(f"Could not parse wheel metadata from {path.name}")
+
+        return PackageSpec(
+            name=canonicalize_name(metadata.name),
+            version=metadata.version,
+            file_name=path.name,
+            sha256=_generate_package_hash(path),
+            package_type="package",
+            install_dir="site",
+            imports=parse_top_level_import_name(path),
+            depends=_wheel_depends(metadata, marker_env),
+        )
+
     def update_sha256(self, path: Path) -> "PackageSpec":
         """Update the sha256 hash for a package."""
         self.sha256 = _generate_package_hash(path)

pyodide_lock/utils.py

@@ -1,11 +1,32 @@
 import hashlib
 import logging
+import re
+import sys
 import zipfile
 from collections import deque
 from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pkginfo import Distribution
 
 logger = logging.getLogger(__name__)
 
+#: the last-observed state of ``packaging.markers.default_environment`` in ``pyodide``
+_PYODIDE_MARKER_ENV = {
+    "implementation_name": "cpython",
+    "implementation_version": "3.11.3",
+    "os_name": "posix",
+    "platform_machine": "wasm32",
+    "platform_release": "3.1.45",
+    "platform_system": "Emscripten",
+    "platform_version": "#1",
+    "python_full_version": "3.11.3",
+    "platform_python_implementation": "CPython",
+    "python_version": "3.11",
+    "sys_platform": "emscripten",
+}
+
 
 def parse_top_level_import_name(whlfile: Path) -> list[str] | None:
     """
@@ -74,3 +95,38 @@ def _generate_package_hash(full_path: Path) -> str:
         while chunk := f.read(4096):
             sha256_hash.update(chunk)
     return sha256_hash.hexdigest()
+
+
+def _wheel_depends(
+    metadata: "Distribution", marker_env: None | dict[str, str] = None
+) -> list[str]:
+    """Get the normalized runtime distribution dependencies from wheel metadata.
+
+    ``marker_env`` is an optional dictionary of platform information, used to find
+    platform-specific requirements as per PEP 508.
+
+    https://peps.python.org/pep-0508
+
+    An accurate enumeration can be generated inside the target pyodide environment
+    such as the example below:
+
+    .. code:
+
+        from packaging.markers import default_environment
+        print(default_enviroment())
+
+    """
+    from packaging.requirements import Requirement
+    from packaging.utils import canonicalize_name
+
+    depends: list[str] = []
+
+    env = dict({} if "pyodide" in sys.modules else _PYODIDE_MARKER_ENV)
+    env.update(marker_env or {})
+
+    for dep_str in metadata.requires_dist:
+        req = Requirement(re.sub(r";$", "", dep_str))
+        if req.marker is None or req.marker.evaluate(env):
+            depends += [canonicalize_name(req.name)]
+
+    return sorted(set(depends))

pyproject.toml

@@ -21,14 +21,29 @@ classifiers = [
 dynamic = ["version"]
 
 [project.optional-dependencies]
+wheel = [
+    "pkginfo",
+    "packaging",
+]
 dev = [
-    "pytest", "pytest-cov"
+    "pytest",
+    "pytest-cov",
+    "build",
+    # from wheel
+    "pkginfo",
+    "packaging",
 ]
 
 [project.urls]
 "Homepage" = "https://github.com/pyodide/pyodide-lock"
 "Bug Tracker" = "https://github.com/pyodide/pyodide-lock/issues"
 
+[tool.coverage.report]
+exclude_lines = [
+  "pragma: no cover",
+  "if TYPE_CHECKING:",
+]
+
 [tool.hatch.version]
 source = "vcs"
 

tests/test_wheel.py

@@ -0,0 +1,100 @@
+import zipfile
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pkginfo
+import pytest
+
+from pyodide_lock import PackageSpec
+from pyodide_lock.utils import (
+    _PYODIDE_MARKER_ENV as _ENV,
+)
+from pyodide_lock.utils import (
+    _generate_package_hash,
+    _wheel_depends,
+)
+
+if TYPE_CHECKING:
+    TDepExamples = dict[tuple[str], list[str]]
+
+HERE = Path(__file__).parent
+DIST = HERE.parent / "dist"
+WHEEL = next(DIST.glob("*.whl")) if DIST.exists() else None
+
+_ENV_NUM = {k: v for k, v in _ENV.items() if v[0] in "0123456789"}
+
+# from https://peps.python.org/pep-0508/#examples
+PEP_0508_EXAMPLES: "TDepExamples" = {
+    ('requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < "2.7"',): [],
+    ('argparse;python_version<"2.7"',): [],
+}
+MARKER_EXAMPLES: "TDepExamples" = {
+    (f'Expected ; {k} == "{v}"',): ["expected"] for k, v in _ENV.items()
+}
+NOT_MARKER_EXAMPLES: "TDepExamples" = {
+    (f'Not.expected ; {k} != "{v}"',): [] for k, v in _ENV.items()
+}
+NUM_MARKER_EXAMPLES: "TDepExamples" = {
+    (f'Expected ; {k} >= "{v}"',): ["expected"] for k, v in _ENV_NUM.items()
+}
+NOT_NUM_MARKER_EXAMPLES: "TDepExamples" = {
+    (f'Not-expected ; {k} < "{v}"',): [] for k, v in _ENV_NUM.items()
+}
+
+
+@pytest.mark.skipif(WHEEL is None, reason="wheel test requires a built wheel")
+def test_self_wheel():
+    assert WHEEL is not None
+
+    spec = PackageSpec.from_wheel(WHEEL).model_dump_json(indent=2)
+
+    expected = PackageSpec(
+        name="pyodide-lock",
+        version=WHEEL.name.split("-")[1],
+        file_name=WHEEL.name,
+        install_dir="site",
+        sha256=_generate_package_hash(WHEEL),
+        package_type="package",
+        imports=["pyodide_lock"],
+        depends=["pydantic"],
+        unvendored_tests=False,
+        shared_library=False,
+    ).model_dump_json(indent=2)
+
+    assert spec == expected
+
+
+def test_not_wheel(tmp_path):
+    wheel = tmp_path / "not-a-wheel-1.0.0-py3-none-any.whl"
+    with zipfile.ZipFile(wheel, "w") as whlzip:
+        whlzip.writestr("README.md", data="Not a wheel")
+
+    with pytest.raises(RuntimeError, match="metadata"):
+        PackageSpec.from_wheel(wheel)
+
+
+@pytest.mark.parametrize(
+    "requires_dist,depends",
+    [
+        *PEP_0508_EXAMPLES.items(),
+        *MARKER_EXAMPLES.items(),
+        *NOT_MARKER_EXAMPLES.items(),
+        *NUM_MARKER_EXAMPLES.items(),
+        *NOT_NUM_MARKER_EXAMPLES.items(),
+        # normalized names
+        (("PyYAML",), ["pyyaml"]),
+        (("pyyaml",), ["pyyaml"]),
+        (("pyyaml", "PyYAML"), ["pyyaml"]),
+        (("ruamel-yaml",), ["ruamel-yaml"]),
+        (("ruamel.yaml",), ["ruamel-yaml"]),
+        (("ruamel.yaml", "ruamel-yaml"), ["ruamel-yaml"]),
+        (("ruamel.yaml.jinja2",), ["ruamel-yaml-jinja2"]),
+    ],
+)
+def test_wheel_depends(requires_dist: tuple[str], depends: list[str]) -> None:
+    metadata = pkginfo.Distribution()
+    metadata.name = "foo"
+    metadata.requires_dist = requires_dist
+    assert (
+        _wheel_depends(metadata) == depends
+    ), f"{requires_dist} does not yield {depends}"