From 0454cc82bccad04f88052f2ab9dfe75d1545310b Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 16 Sep 2023 11:34:23 -0500 Subject: [PATCH 1/2] add PackageSpec.from_wheel --- .github/workflows/main.yml | 3 ++ CHANGELOG.md | 2 ++ pyodide_lock/__init__.py | 3 +- pyodide_lock/spec.py | 36 ++++++++++++++++++++++- pyodide_lock/utils.py | 60 ++++++++++++++++++++++++++++++++++++++ pyproject.toml | 17 ++++++++++- tests/test_wheel.py | 42 ++++++++++++++++++++++++++ 7 files changed, 160 insertions(+), 3 deletions(-) create mode 100644 tests/test_wheel.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e9a5a5..971d4bf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,6 +24,9 @@ jobs: run: | python -m pip install --upgrade pip pip install -e ".[dev]" + - name: Build + run: | + pyproject-build - name: Test with pytest run: | pytest --cov=. --cov-report=xml diff --git a/CHANGELOG.md b/CHANGELOG.md index eb4b2da..49c8902 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Adds `PackageSpec.from_wheel` for generating a package spec from a `.whl` file + [#18](https://github.com/pyodide/pyodide-lock/pull/18) - Adds `parse_top_level_import_name` for finding importable names in `.whl` files [#17](https://github.com/pyodide/pyodide-lock/pull/17) diff --git a/pyodide_lock/__init__.py b/pyodide_lock/__init__.py index febce82..c6d10ba 100644 --- a/pyodide_lock/__init__.py +++ b/pyodide_lock/__init__.py @@ -1,7 +1,8 @@ -from .spec import PyodideLockSpec +from .spec import PackageSpec, PyodideLockSpec from .utils import parse_top_level_import_name __all__ = [ "PyodideLockSpec", + "PackageSpec", "parse_top_level_import_name", ] diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index 84f76be..8397c34 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -4,7 +4,12 @@ from pydantic import BaseModel, ConfigDict -from .utils import _generate_package_hash +from .utils import ( + _generate_package_hash, + _normalized_name, + _wheel_depends, + parse_top_level_import_name, +) class InfoSpec(BaseModel): @@ -33,6 +38,35 @@ class PackageSpec(BaseModel): # This field is deprecated shared_library: bool = False + @classmethod + def from_wheel( + cls, + path: Path, + marker_env: None | dict[str, str] = None, + ) -> "PackageSpec": + """Build a package spec from an on-disk wheel. + + This currently assumes a "simple" noarch wheel: more complex packages + may require further postprocessing. + """ + import pkginfo + + metadata = pkginfo.get_metadata(str(path)) + + if not metadata: + raise RuntimeError(f"Could not parse wheel metadata from {path.name}") + + return PackageSpec( + name=_normalized_name(metadata.name), + version=metadata.version, + file_name=path.name, + sha256=_generate_package_hash(path), + package_type="package", + install_dir="site", + imports=parse_top_level_import_name(path), + depends=_wheel_depends(metadata, marker_env), + ) + def update_sha256(self, path: Path) -> "PackageSpec": """Update the sha256 hash for a package.""" self.sha256 = _generate_package_hash(path) diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index 03a24e6..b7ceca8 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -1,11 +1,32 @@ import hashlib import logging +import re +import sys import zipfile from collections import deque from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pkginfo import Distribution logger = logging.getLogger(__name__) +#: the last-observed state of ``packaging.markers.default_environment`` in ``pyodide`` +_PYODIDE_MARKER_ENV = { + "implementation_name": "cpython", + "implementation_version": "3.11.3", + "os_name": "posix", + "platform_machine": "wasm32", + "platform_release": "3.1.45", + "platform_system": "Emscripten", + "platform_version": "#1", + "python_full_version": "3.11.3", + "platform_python_implementation": "CPython", + "python_version": "3.11", + "sys_platform": "emscripten", +} + def parse_top_level_import_name(whlfile: Path) -> list[str] | None: """ @@ -74,3 +95,42 @@ def _generate_package_hash(full_path: Path) -> str: while chunk := f.read(4096): sha256_hash.update(chunk) return sha256_hash.hexdigest() + + +def _normalized_name(raw_name: str) -> str: + """Get a PEP 503 normalized name for a python package. + + https://peps.python.org/pep-0503/#normalized-names + """ + return re.sub(r"[-_.]+", "-", raw_name).lower() + + +def _wheel_depends( + metadata: "Distribution", marker_env: None | dict[str, str] = None +) -> list[str]: + """Get the normalized runtime distribution dependencies from wheel metadata. + + ``marker_env`` is an optional dictionary of platform information, used to find + platform-specific requirments. + + An accurate enumeration can be generated inside the target pyodide environment + such as the example below: + + .. code: + + from packaging.markers import default_environment + print(default_enviroment()) + """ + from packaging.requirements import Requirement + + depends: list[str] = [] + + env = {} if "pyodide" in sys.modules else _PYODIDE_MARKER_ENV + env.update(marker_env or {}) + + for dep_str in metadata.requires_dist: + req = Requirement(re.sub(r";$", "", dep_str)) + if req.marker is None or req.marker.evaluate(env): + depends += [_normalized_name(req.name)] + + return sorted(set(depends)) diff --git a/pyproject.toml b/pyproject.toml index 1459a63..ebd6d0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,14 +21,29 @@ classifiers = [ dynamic = ["version"] [project.optional-dependencies] +wheel = [ + "pkginfo", + "packaging", +] dev = [ - "pytest", "pytest-cov" + "pytest", + "pytest-cov", + "build", + # from wheel + "pkginfo", + "packaging", ] [project.urls] "Homepage" = "https://github.com/pyodide/pyodide-lock" "Bug Tracker" = "https://github.com/pyodide/pyodide-lock/issues" +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", +] + [tool.hatch.version] source = "vcs" diff --git a/tests/test_wheel.py b/tests/test_wheel.py new file mode 100644 index 0000000..8c9eee4 --- /dev/null +++ b/tests/test_wheel.py @@ -0,0 +1,42 @@ +import zipfile +from pathlib import Path + +import pytest + +from pyodide_lock import PackageSpec +from pyodide_lock.utils import _generate_package_hash + +HERE = Path(__file__).parent +DIST = HERE.parent / "dist" +WHEEL = next(DIST.glob("*.whl")) if DIST.exists() else None + + +@pytest.mark.skipif(WHEEL is None, reason="wheel test requires a built wheel") +def test_self_wheel(): + assert WHEEL is not None + + spec = PackageSpec.from_wheel(WHEEL).model_dump_json(indent=2) + + expected = PackageSpec( + name="pyodide-lock", + version=WHEEL.name.split("-")[1], + file_name=WHEEL.name, + install_dir="site", + sha256=_generate_package_hash(WHEEL), + package_type="package", + imports=["pyodide_lock"], + depends=["pydantic"], + unvendored_tests=False, + shared_library=False, + ).model_dump_json(indent=2) + + assert spec == expected + + +def test_not_wheel(tmp_path): + wheel = tmp_path / "not-a-wheel-1.0.0-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w") as whlzip: + whlzip.writestr("README.md", data="Not a wheel") + + with pytest.raises(RuntimeError, match="metadata"): + PackageSpec.from_wheel(wheel) From b01b33b150e17480450700c0a6882239c671ccb7 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Tue, 26 Sep 2023 09:21:18 -0500 Subject: [PATCH 2/2] rework name normalization --- pyodide_lock/spec.py | 4 +-- pyodide_lock/utils.py | 18 +++++-------- tests/test_wheel.py | 60 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index 8397c34..ac860b2 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -6,7 +6,6 @@ from .utils import ( _generate_package_hash, - _normalized_name, _wheel_depends, parse_top_level_import_name, ) @@ -50,6 +49,7 @@ def from_wheel( may require further postprocessing. """ import pkginfo + from packaging.utils import canonicalize_name metadata = pkginfo.get_metadata(str(path)) @@ -57,7 +57,7 @@ def from_wheel( raise RuntimeError(f"Could not parse wheel metadata from {path.name}") return PackageSpec( - name=_normalized_name(metadata.name), + name=canonicalize_name(metadata.name), version=metadata.version, file_name=path.name, sha256=_generate_package_hash(path), diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index b7ceca8..dfbefd3 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -97,21 +97,15 @@ def _generate_package_hash(full_path: Path) -> str: return sha256_hash.hexdigest() -def _normalized_name(raw_name: str) -> str: - """Get a PEP 503 normalized name for a python package. - - https://peps.python.org/pep-0503/#normalized-names - """ - return re.sub(r"[-_.]+", "-", raw_name).lower() - - def _wheel_depends( metadata: "Distribution", marker_env: None | dict[str, str] = None ) -> list[str]: """Get the normalized runtime distribution dependencies from wheel metadata. ``marker_env`` is an optional dictionary of platform information, used to find - platform-specific requirments. + platform-specific requirements as per PEP 508. + + https://peps.python.org/pep-0508 An accurate enumeration can be generated inside the target pyodide environment such as the example below: @@ -120,17 +114,19 @@ def _wheel_depends( from packaging.markers import default_environment print(default_enviroment()) + """ from packaging.requirements import Requirement + from packaging.utils import canonicalize_name depends: list[str] = [] - env = {} if "pyodide" in sys.modules else _PYODIDE_MARKER_ENV + env = dict({} if "pyodide" in sys.modules else _PYODIDE_MARKER_ENV) env.update(marker_env or {}) for dep_str in metadata.requires_dist: req = Requirement(re.sub(r";$", "", dep_str)) if req.marker is None or req.marker.evaluate(env): - depends += [_normalized_name(req.name)] + depends += [canonicalize_name(req.name)] return sorted(set(depends)) diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 8c9eee4..a008bb8 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -1,15 +1,46 @@ import zipfile from pathlib import Path +from typing import TYPE_CHECKING +import pkginfo import pytest from pyodide_lock import PackageSpec -from pyodide_lock.utils import _generate_package_hash +from pyodide_lock.utils import ( + _PYODIDE_MARKER_ENV as _ENV, +) +from pyodide_lock.utils import ( + _generate_package_hash, + _wheel_depends, +) + +if TYPE_CHECKING: + TDepExamples = dict[tuple[str], list[str]] HERE = Path(__file__).parent DIST = HERE.parent / "dist" WHEEL = next(DIST.glob("*.whl")) if DIST.exists() else None +_ENV_NUM = {k: v for k, v in _ENV.items() if v[0] in "0123456789"} + +# from https://peps.python.org/pep-0508/#examples +PEP_0508_EXAMPLES: "TDepExamples" = { + ('requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < "2.7"',): [], + ('argparse;python_version<"2.7"',): [], +} +MARKER_EXAMPLES: "TDepExamples" = { + (f'Expected ; {k} == "{v}"',): ["expected"] for k, v in _ENV.items() +} +NOT_MARKER_EXAMPLES: "TDepExamples" = { + (f'Not.expected ; {k} != "{v}"',): [] for k, v in _ENV.items() +} +NUM_MARKER_EXAMPLES: "TDepExamples" = { + (f'Expected ; {k} >= "{v}"',): ["expected"] for k, v in _ENV_NUM.items() +} +NOT_NUM_MARKER_EXAMPLES: "TDepExamples" = { + (f'Not-expected ; {k} < "{v}"',): [] for k, v in _ENV_NUM.items() +} + @pytest.mark.skipif(WHEEL is None, reason="wheel test requires a built wheel") def test_self_wheel(): @@ -40,3 +71,30 @@ def test_not_wheel(tmp_path): with pytest.raises(RuntimeError, match="metadata"): PackageSpec.from_wheel(wheel) + + +@pytest.mark.parametrize( + "requires_dist,depends", + [ + *PEP_0508_EXAMPLES.items(), + *MARKER_EXAMPLES.items(), + *NOT_MARKER_EXAMPLES.items(), + *NUM_MARKER_EXAMPLES.items(), + *NOT_NUM_MARKER_EXAMPLES.items(), + # normalized names + (("PyYAML",), ["pyyaml"]), + (("pyyaml",), ["pyyaml"]), + (("pyyaml", "PyYAML"), ["pyyaml"]), + (("ruamel-yaml",), ["ruamel-yaml"]), + (("ruamel.yaml",), ["ruamel-yaml"]), + (("ruamel.yaml", "ruamel-yaml"), ["ruamel-yaml"]), + (("ruamel.yaml.jinja2",), ["ruamel-yaml-jinja2"]), + ], +) +def test_wheel_depends(requires_dist: tuple[str], depends: list[str]) -> None: + metadata = pkginfo.Distribution() + metadata.name = "foo" + metadata.requires_dist = requires_dist + assert ( + _wheel_depends(metadata) == depends + ), f"{requires_dist} does not yield {depends}"