From 34bab06b67ccfa1faf5b46bb877bbf4e8d689691 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 22 Sep 2023 13:50:26 +0100 Subject: [PATCH 01/22] added cli with option to add a list of wheels --- pyodide_lock/cli/__init__.py | 0 pyodide_lock/cli/lockfile.py | 20 +++++ pyodide_lock/spec.py | 139 ++++++++++++++++++++++++++++++++++- pyodide_lock/utils.py | 63 ++++++++++++++++ pyproject.toml | 7 +- 5 files changed, 226 insertions(+), 3 deletions(-) create mode 100644 pyodide_lock/cli/__init__.py create mode 100644 pyodide_lock/cli/lockfile.py diff --git a/pyodide_lock/cli/__init__.py b/pyodide_lock/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyodide_lock/cli/lockfile.py b/pyodide_lock/cli/lockfile.py new file mode 100644 index 0000000..1827925 --- /dev/null +++ b/pyodide_lock/cli/lockfile.py @@ -0,0 +1,20 @@ +import typer +from pathlib import Path +from typing import List +from typing_extensions import Annotated + +from ..spec import PyodideLockSpec + +main=typer.Typer() + +@main.command() +def add_wheels( + wheels: Annotated[List[Path],typer.Argument(help="list of wheels to add to the lockfile",default="[]")], + in_lockfile: Path = "pyodide-lock.json", + out_lockfile: Path = "pyodide-lock-new.json", +): + """Add a set of wheels to an existing pyodide-lock.json + """ + sp=PyodideLockSpec.from_json(in_lockfile) + sp.add_wheels(wheels) + sp.to_json(out_lockfile) diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index 84f76be..ca8e7e0 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -4,7 +4,8 @@ from pydantic import BaseModel, ConfigDict -from .utils import _generate_package_hash +from .utils import _generate_package_hash, parse_top_level_import_name, get_wheel_dependencies + class InfoSpec(BaseModel): @@ -57,7 +58,11 @@ def from_json(cls, path: Path) -> "PyodideLockSpec": def to_json(self, path: Path, indent: int | None = None) -> None: """Write the lock spec to a json file.""" with path.open("w") as fh: - json.dump(self.model_dump(), fh, indent=indent) + # old vs new pydantic + if hasattr(self,"model_dump"): + json.dump(self.model_dump(), fh, indent=indent) + else: + json.dump(self.dict(), fh, indent=indent) def check_wheel_filenames(self) -> None: """Check that the package name and version are consistent in wheel filenames""" @@ -91,3 +96,133 @@ def check_wheel_filenames(self) -> None: for name, errs in errors.items() ) raise ValueError(error_msg) + + def add_wheels( + self, + wheel_files: list[Path], + base_path: Path | None = None, + base_url: str = "", + fix_dependencies: bool = True, + ) -> None: + """Add a list of wheel files to this pyodide-lock.json + + Args: + wheel_files (list[Path]): A list of wheel files to import. + base_path (Path | None, optional): + Filenames are stored relative to this base path. By default the + filename is stored relative to the path of the first wheel file + in the list. + + base_url (str, optional): + The base URL stored in the pyodide-lock.json. By default this is empty + which means that wheels must be stored in the same folder as the core pyodide + packages you are using. If you want to store your custom wheels somewhere + else, set this base_url to point to it. + + fix_dependencies (bool, optional): + If this is True, then the dependency lists for each + package will be populated with any packages which are available in this pyodide + distribution. Defaults to True + """ + if len(wheel_files) <= 0: + return + if base_path == None: + base_path = wheel_files[0].parent + + from packaging.utils import canonicalize_name + from packaging.version import parse as version_parse + + target_python = version_parse(self.info.python) + python_binary_tag = f"cp{target_python.major}{target_python.minor}" + python_pure_tags=[f"py2.py{target_python.major}",f"py{target_python.major}",f"py{target_python.major}{target_python.minor}"] + + target_platform=self.info.platform+"_"+self.info.arch + + new_packages = {} + new_package_wheels ={} + for f in wheel_files: + split_name = f.stem.split("-") + name = canonicalize_name(split_name[0]) + version = split_name[1] + python_tag = split_name[-3] + abi_tag = split_name[-2] + platform_tag = split_name[-1] + + if platform_tag=="any": + if python_tag not in python_pure_tags: + raise ValueError( + f"Wheel {f} is built for incorrect python version {python_tag}, this lockfile expects {python_binary_tag} or one of {python_pure_tags}" + ) + elif platform_tag != target_platform: + raise ValueError( + f"Wheel {f} is built for incorrect platform {platform_tag}, this lockfile expects {target_platform}" + ) + else: + if python_tag != python_binary_tag: + raise ValueError( + f"Wheel {f} is built for incorrect python version {python_tag}, this lockfile expects {python_binary_tag}" + ) + + file_name = base_url + str(f.relative_to(base_path)) + install_dir = "site" + package_type = "package" + sha256 = _generate_package_hash(f) + imports = parse_top_level_import_name(f) + + new_packages[name] = PackageSpec( + name=name, + version=version, + install_dir=install_dir, + file_name=file_name, + package_type=package_type, + sha256=sha256, + imports=imports, + depends=[], + ) + new_package_wheels[name]=f + # now fix up the dependencies for each of our new packages + # n.b. this assumes existing packages have correct dependencies, + # which is probably a good assumption. + + requirements_with_extras = [] + for package in new_packages.values(): + # add any requirements to the list of packages + our_depends=[] + wheel_file=new_package_wheels[package.name] + requirements = get_wheel_dependencies(wheel_file, package.name) + for r in requirements: + req_marker = r.marker + req_name = canonicalize_name(r.name) + if req_marker is not None: + if not req_marker.evaluate({"sys_platform":"emscripten","platform_system":"Emscripten"}): + # not used in pyodide / emscripten + # or optional requirement + continue + if r.extras: + # this requirement has some extras, we need to check that the dependency package + # depends on whatever needs these extras also. + requirements_with_extras.append(r) + if req_name in new_packages or req_name in self.packages: + our_depends.append(req_name) + package.depends=our_depends + + while len(requirements_with_extras)!=0: + extra_req=requirements_with_extras.pop() + extra_package_name = canonicalize_name(r.name) + if extra_package_name in new_packages: + package=new_packages[extra_package_name] + our_depends=package.depends + wheel_file=new_package_wheels[package.name] + requirements = get_wheel_dependencies(wheel_file, package.name) + for extra in extra_req.extras: + for r in requirements: + req_marker = r.marker + req_name = canonicalize_name(r.name) + if req_marker is not None: + if req_marker.evaluate({"sys_platform":"emscripten","platform_system":"Emscripten","extra":extra}): + if req_name in new_packages or req_name in self.packages: + our_depends.append(req_name) + if r.extras: + requirements_with_extras.append(r) + package.depends=our_depends + self.packages.update(new_packages) diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index 03a24e6..2595321 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -3,6 +3,13 @@ import zipfile from collections import deque from pathlib import Path +from zipfile import ZipFile +from email.parser import BytesParser + +from packaging.utils import canonicalize_name as canonicalize_package_name +from packaging.utils import parse_wheel_filename +from packaging.requirements import Requirement + logger = logging.getLogger(__name__) @@ -20,6 +27,14 @@ def parse_top_level_import_name(whlfile: Path) -> list[str] | None: whlzip = zipfile.Path(whlfile) + # if there is a directory with .dist_info at the end with a top_level.txt file + # then just use that + for subdir in whlzip.iterdir(): + if subdir.name.endswith(".dist-info"): + top_level_path=subdir / "top_level.txt" + if top_level_path.exists(): + return top_level_path.read_text().splitlines() + # If there is no top_level.txt file, we will find top level imports by # 1) a python file on a top-level directory # 2) a sub directory with __init__.py @@ -74,3 +89,51 @@ def _generate_package_hash(full_path: Path) -> str: while chunk := f.read(4096): sha256_hash.update(chunk) return sha256_hash.hexdigest() + +def get_wheel_dependencies(wheel_path: Path, pkg_name: str) -> list[str]: + deps = [] + if not wheel_path.name.endswith(".whl"): + raise RuntimeError(f"{wheel_path} is not a wheel file.") + with ZipFile(wheel_path, mode="r") as wheel: + dist_info_dir = get_wheel_dist_info_dir(wheel,pkg_name) + metadata_path = f"{dist_info_dir}/METADATA" + p = BytesParser() + headers = p.parse(wheel.open(metadata_path), headersonly=True) + requires = headers.get_all("Requires-Dist", failobj=[]) + for r in requires: + deps.append(Requirement(r)) + return deps + +def get_wheel_dist_info_dir(wheel: ZipFile, pkg_name: str) -> str: + """Returns the path of the contained .dist-info directory. + + Raises an Exception if the directory is not found, more than + one is found, or it does not match the provided `pkg_name`. + + Adapted from: + https://github.com/pypa/pip/blob/ea727e4d6ab598f34f97c50a22350febc1214a97/src/pip/_internal/utils/wheel.py#L38 + """ + + # Zip file path separators must be / + subdirs = {name.split("/", 1)[0] for name in wheel.namelist()} + info_dirs = [subdir for subdir in subdirs if subdir.endswith(".dist-info")] + + if len(info_dirs) == 0: + raise Exception(f".dist-info directory not found for {pkg_name}") + + if len(info_dirs) > 1: + raise Exception( + f"multiple .dist-info directories found for {pkg_name}: {', '.join(info_dirs)}" + ) + + (info_dir,) = info_dirs + + info_dir_name = canonicalize_package_name(info_dir) + canonical_name = canonicalize_package_name(pkg_name) + + if not info_dir_name.startswith(canonical_name): + raise Exception( + f".dist-info directory {info_dir!r} does not start with {canonical_name!r}" + ) + + return info_dir diff --git a/pyproject.toml b/pyproject.toml index 1459a63..e676285 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,9 @@ description = "Tooling to manage the `pyodide-lock.json` file" readme = "README.md" requires-python = ">=3.10" dependencies = [ - "pydantic" + "pydantic", + "packaging", + "pyodide-cli" ] classifiers = [ "Programming Language :: Python :: 3", @@ -60,3 +62,6 @@ select = [ [tool.pytest.ini_options] addopts = ''' --doctest-modules''' + +[project.entry-points."pyodide.cli"] +lockfile = "pyodide_lock.cli.lockfile:main" From 5a1a70ad16874aed8788db3d292d0c9a31823d6c Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 22 Sep 2023 13:50:56 +0100 Subject: [PATCH 02/22] black --- pyodide_lock/cli/lockfile.py | 13 +++++--- pyodide_lock/spec.py | 60 +++++++++++++++++++++++------------- pyodide_lock/utils.py | 8 +++-- 3 files changed, 52 insertions(+), 29 deletions(-) diff --git a/pyodide_lock/cli/lockfile.py b/pyodide_lock/cli/lockfile.py index 1827925..39f131d 100644 --- a/pyodide_lock/cli/lockfile.py +++ b/pyodide_lock/cli/lockfile.py @@ -5,16 +5,19 @@ from ..spec import PyodideLockSpec -main=typer.Typer() +main = typer.Typer() + @main.command() def add_wheels( - wheels: Annotated[List[Path],typer.Argument(help="list of wheels to add to the lockfile",default="[]")], + wheels: Annotated[ + List[Path], + typer.Argument(help="list of wheels to add to the lockfile", default="[]"), + ], in_lockfile: Path = "pyodide-lock.json", out_lockfile: Path = "pyodide-lock-new.json", ): - """Add a set of wheels to an existing pyodide-lock.json - """ - sp=PyodideLockSpec.from_json(in_lockfile) + """Add a set of wheels to an existing pyodide-lock.json""" + sp = PyodideLockSpec.from_json(in_lockfile) sp.add_wheels(wheels) sp.to_json(out_lockfile) diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index ca8e7e0..7882637 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -4,8 +4,11 @@ from pydantic import BaseModel, ConfigDict -from .utils import _generate_package_hash, parse_top_level_import_name, get_wheel_dependencies - +from .utils import ( + _generate_package_hash, + parse_top_level_import_name, + get_wheel_dependencies, +) class InfoSpec(BaseModel): @@ -59,7 +62,7 @@ def to_json(self, path: Path, indent: int | None = None) -> None: """Write the lock spec to a json file.""" with path.open("w") as fh: # old vs new pydantic - if hasattr(self,"model_dump"): + if hasattr(self, "model_dump"): json.dump(self.model_dump(), fh, indent=indent) else: json.dump(self.dict(), fh, indent=indent) @@ -134,12 +137,16 @@ def add_wheels( target_python = version_parse(self.info.python) python_binary_tag = f"cp{target_python.major}{target_python.minor}" - python_pure_tags=[f"py2.py{target_python.major}",f"py{target_python.major}",f"py{target_python.major}{target_python.minor}"] + python_pure_tags = [ + f"py2.py{target_python.major}", + f"py{target_python.major}", + f"py{target_python.major}{target_python.minor}", + ] - target_platform=self.info.platform+"_"+self.info.arch + target_platform = self.info.platform + "_" + self.info.arch new_packages = {} - new_package_wheels ={} + new_package_wheels = {} for f in wheel_files: split_name = f.stem.split("-") name = canonicalize_name(split_name[0]) @@ -148,7 +155,7 @@ def add_wheels( abi_tag = split_name[-2] platform_tag = split_name[-1] - if platform_tag=="any": + if platform_tag == "any": if python_tag not in python_pure_tags: raise ValueError( f"Wheel {f} is built for incorrect python version {python_tag}, this lockfile expects {python_binary_tag} or one of {python_pure_tags}" @@ -179,7 +186,7 @@ def add_wheels( imports=imports, depends=[], ) - new_package_wheels[name]=f + new_package_wheels[name] = f # now fix up the dependencies for each of our new packages # n.b. this assumes existing packages have correct dependencies, # which is probably a good assumption. @@ -187,14 +194,16 @@ def add_wheels( requirements_with_extras = [] for package in new_packages.values(): # add any requirements to the list of packages - our_depends=[] - wheel_file=new_package_wheels[package.name] + our_depends = [] + wheel_file = new_package_wheels[package.name] requirements = get_wheel_dependencies(wheel_file, package.name) for r in requirements: req_marker = r.marker req_name = canonicalize_name(r.name) if req_marker is not None: - if not req_marker.evaluate({"sys_platform":"emscripten","platform_system":"Emscripten"}): + if not req_marker.evaluate( + {"sys_platform": "emscripten", "platform_system": "Emscripten"} + ): # not used in pyodide / emscripten # or optional requirement continue @@ -204,25 +213,34 @@ def add_wheels( requirements_with_extras.append(r) if req_name in new_packages or req_name in self.packages: our_depends.append(req_name) - package.depends=our_depends - - while len(requirements_with_extras)!=0: - extra_req=requirements_with_extras.pop() + package.depends = our_depends + + while len(requirements_with_extras) != 0: + extra_req = requirements_with_extras.pop() extra_package_name = canonicalize_name(r.name) if extra_package_name in new_packages: - package=new_packages[extra_package_name] - our_depends=package.depends - wheel_file=new_package_wheels[package.name] + package = new_packages[extra_package_name] + our_depends = package.depends + wheel_file = new_package_wheels[package.name] requirements = get_wheel_dependencies(wheel_file, package.name) for extra in extra_req.extras: for r in requirements: req_marker = r.marker req_name = canonicalize_name(r.name) if req_marker is not None: - if req_marker.evaluate({"sys_platform":"emscripten","platform_system":"Emscripten","extra":extra}): - if req_name in new_packages or req_name in self.packages: + if req_marker.evaluate( + { + "sys_platform": "emscripten", + "platform_system": "Emscripten", + "extra": extra, + } + ): + if ( + req_name in new_packages + or req_name in self.packages + ): our_depends.append(req_name) if r.extras: requirements_with_extras.append(r) - package.depends=our_depends + package.depends = our_depends self.packages.update(new_packages) diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index 2595321..eceb39f 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -31,10 +31,10 @@ def parse_top_level_import_name(whlfile: Path) -> list[str] | None: # then just use that for subdir in whlzip.iterdir(): if subdir.name.endswith(".dist-info"): - top_level_path=subdir / "top_level.txt" + top_level_path = subdir / "top_level.txt" if top_level_path.exists(): return top_level_path.read_text().splitlines() - + # If there is no top_level.txt file, we will find top level imports by # 1) a python file on a top-level directory # 2) a sub directory with __init__.py @@ -90,12 +90,13 @@ def _generate_package_hash(full_path: Path) -> str: sha256_hash.update(chunk) return sha256_hash.hexdigest() + def get_wheel_dependencies(wheel_path: Path, pkg_name: str) -> list[str]: deps = [] if not wheel_path.name.endswith(".whl"): raise RuntimeError(f"{wheel_path} is not a wheel file.") with ZipFile(wheel_path, mode="r") as wheel: - dist_info_dir = get_wheel_dist_info_dir(wheel,pkg_name) + dist_info_dir = get_wheel_dist_info_dir(wheel, pkg_name) metadata_path = f"{dist_info_dir}/METADATA" p = BytesParser() headers = p.parse(wheel.open(metadata_path), headersonly=True) @@ -104,6 +105,7 @@ def get_wheel_dependencies(wheel_path: Path, pkg_name: str) -> list[str]: deps.append(Requirement(r)) return deps + def get_wheel_dist_info_dir(wheel: ZipFile, pkg_name: str) -> str: """Returns the path of the contained .dist-info directory. From a8cea59222d343c95a56cc72d86cab46b065362d Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 22 Sep 2023 13:53:50 +0100 Subject: [PATCH 03/22] removed option to not fix deps --- pyodide_lock/spec.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index 7882637..cd6d91c 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -105,7 +105,6 @@ def add_wheels( wheel_files: list[Path], base_path: Path | None = None, base_url: str = "", - fix_dependencies: bool = True, ) -> None: """Add a list of wheel files to this pyodide-lock.json @@ -121,11 +120,6 @@ def add_wheels( which means that wheels must be stored in the same folder as the core pyodide packages you are using. If you want to store your custom wheels somewhere else, set this base_url to point to it. - - fix_dependencies (bool, optional): - If this is True, then the dependency lists for each - package will be populated with any packages which are available in this pyodide - distribution. Defaults to True """ if len(wheel_files) <= 0: return From 55d7fdd346ea4ac14bfad0d51b50d7723c9d56a7 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 22 Sep 2023 13:59:01 +0100 Subject: [PATCH 04/22] error handling for missing dependencies --- pyodide_lock/spec.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index cd6d91c..7353b8f 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -98,7 +98,7 @@ def check_wheel_filenames(self) -> None: f"{name}:\n - " + "\n - ".join(errs) for name, errs in errors.items() ) - raise ValueError(error_msg) + raise RuntimeError(error_msg) def add_wheels( self, @@ -151,16 +151,16 @@ def add_wheels( if platform_tag == "any": if python_tag not in python_pure_tags: - raise ValueError( + raise RuntimeError( f"Wheel {f} is built for incorrect python version {python_tag}, this lockfile expects {python_binary_tag} or one of {python_pure_tags}" ) elif platform_tag != target_platform: - raise ValueError( + raise RuntimeError( f"Wheel {f} is built for incorrect platform {platform_tag}, this lockfile expects {target_platform}" ) else: if python_tag != python_binary_tag: - raise ValueError( + raise RuntimeError( f"Wheel {f} is built for incorrect python version {python_tag}, this lockfile expects {python_binary_tag}" ) @@ -207,6 +207,8 @@ def add_wheels( requirements_with_extras.append(r) if req_name in new_packages or req_name in self.packages: our_depends.append(req_name) + else: + raise RuntimeError(f"Requirement {req_name} is not in this distribution.") package.depends = our_depends while len(requirements_with_extras) != 0: @@ -236,5 +238,7 @@ def add_wheels( our_depends.append(req_name) if r.extras: requirements_with_extras.append(r) + else: + raise RuntimeError(f"Requirement {req_name} is not in this distribution.") package.depends = our_depends self.packages.update(new_packages) From 1876ce1e2ed1a67bd8c4a0981c09ccaa83fa1e93 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 22 Sep 2023 14:10:40 +0100 Subject: [PATCH 05/22] typing --- pyodide_lock/cli/lockfile.py | 4 ++-- pyodide_lock/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyodide_lock/cli/lockfile.py b/pyodide_lock/cli/lockfile.py index 39f131d..d47131e 100644 --- a/pyodide_lock/cli/lockfile.py +++ b/pyodide_lock/cli/lockfile.py @@ -14,8 +14,8 @@ def add_wheels( List[Path], typer.Argument(help="list of wheels to add to the lockfile", default="[]"), ], - in_lockfile: Path = "pyodide-lock.json", - out_lockfile: Path = "pyodide-lock-new.json", + in_lockfile: Path = Path("pyodide-lock.json"), + out_lockfile: Path = Path("pyodide-lock-new.json"), ): """Add a set of wheels to an existing pyodide-lock.json""" sp = PyodideLockSpec.from_json(in_lockfile) diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index eceb39f..e30cc4f 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -91,7 +91,7 @@ def _generate_package_hash(full_path: Path) -> str: return sha256_hash.hexdigest() -def get_wheel_dependencies(wheel_path: Path, pkg_name: str) -> list[str]: +def get_wheel_dependencies(wheel_path: Path, pkg_name: str) -> list[Requirement]: deps = [] if not wheel_path.name.endswith(".whl"): raise RuntimeError(f"{wheel_path} is not a wheel file.") @@ -100,7 +100,7 @@ def get_wheel_dependencies(wheel_path: Path, pkg_name: str) -> list[str]: metadata_path = f"{dist_info_dir}/METADATA" p = BytesParser() headers = p.parse(wheel.open(metadata_path), headersonly=True) - requires = headers.get_all("Requires-Dist", failobj=[]) + requires : list[str] = headers.get_all("Requires-Dist", failobj=[]) for r in requires: deps.append(Requirement(r)) return deps From 9d3990b7c69b7eb39ab9c2d3fb4fdb5dc0776150 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 22 Sep 2023 14:22:24 +0100 Subject: [PATCH 06/22] pre-commit fixes --- pyodide_lock/cli/lockfile.py | 8 +-- pyodide_lock/spec.py | 95 +++++++++++++++++++----------------- pyodide_lock/utils.py | 11 ++--- 3 files changed, 60 insertions(+), 54 deletions(-) diff --git a/pyodide_lock/cli/lockfile.py b/pyodide_lock/cli/lockfile.py index d47131e..3c5120a 100644 --- a/pyodide_lock/cli/lockfile.py +++ b/pyodide_lock/cli/lockfile.py @@ -1,7 +1,7 @@ -import typer from pathlib import Path -from typing import List -from typing_extensions import Annotated +from typing import Annotated + +import typer from ..spec import PyodideLockSpec @@ -11,7 +11,7 @@ @main.command() def add_wheels( wheels: Annotated[ - List[Path], + list[Path], typer.Argument(help="list of wheels to add to the lockfile", default="[]"), ], in_lockfile: Path = Path("pyodide-lock.json"), diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index 7353b8f..02a5339 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -2,12 +2,14 @@ from pathlib import Path from typing import Literal +from packaging.utils import canonicalize_name +from packaging.version import parse as version_parse from pydantic import BaseModel, ConfigDict from .utils import ( _generate_package_hash, - parse_top_level_import_name, get_wheel_dependencies, + parse_top_level_import_name, ) @@ -100,7 +102,7 @@ def check_wheel_filenames(self) -> None: ) raise RuntimeError(error_msg) - def add_wheels( + def add_wheels( # noqa: C901 self, wheel_files: list[Path], base_path: Path | None = None, @@ -116,19 +118,16 @@ def add_wheels( in the list. base_url (str, optional): - The base URL stored in the pyodide-lock.json. By default this is empty - which means that wheels must be stored in the same folder as the core pyodide - packages you are using. If you want to store your custom wheels somewhere - else, set this base_url to point to it. + The base URL stored in the pyodide-lock.json. By default this + is empty which means that wheels must be stored in the same folder + as the core pyodide packages you are using. If you want to store + your custom wheels somewhere else, set this base_url to point to it. """ if len(wheel_files) <= 0: return - if base_path == None: + if base_path is None: base_path = wheel_files[0].parent - from packaging.utils import canonicalize_name - from packaging.version import parse as version_parse - target_python = version_parse(self.info.python) python_binary_tag = f"cp{target_python.major}{target_python.minor}" python_pure_tags = [ @@ -146,22 +145,26 @@ def add_wheels( name = canonicalize_name(split_name[0]) version = split_name[1] python_tag = split_name[-3] - abi_tag = split_name[-2] + split_name[-2] platform_tag = split_name[-1] if platform_tag == "any": if python_tag not in python_pure_tags: raise RuntimeError( - f"Wheel {f} is built for incorrect python version {python_tag}, this lockfile expects {python_binary_tag} or one of {python_pure_tags}" + f"Wheel {f} is built for incorrect python version {python_tag}," + f"this lockfile expects {python_binary_tag} " + f"or one of {python_pure_tags}" ) elif platform_tag != target_platform: raise RuntimeError( - f"Wheel {f} is built for incorrect platform {platform_tag}, this lockfile expects {target_platform}" + f"Wheel {f} is built for incorrect platform {platform_tag}," + f"this lockfile expects {target_platform}" ) else: if python_tag != python_binary_tag: raise RuntimeError( - f"Wheel {f} is built for incorrect python version {python_tag}, this lockfile expects {python_binary_tag}" + f"Wheel {f} is built for incorrect python version {python_tag}," + f" this lockfile expects {python_binary_tag}" ) file_name = base_url + str(f.relative_to(base_path)) @@ -181,6 +184,7 @@ def add_wheels( depends=[], ) new_package_wheels[name] = f + # now fix up the dependencies for each of our new packages # n.b. this assumes existing packages have correct dependencies, # which is probably a good assumption. @@ -202,43 +206,46 @@ def add_wheels( # or optional requirement continue if r.extras: - # this requirement has some extras, we need to check that the dependency package - # depends on whatever needs these extras also. + # this requirement has some extras, we need to check + # that the required package depends on these extras also. requirements_with_extras.append(r) if req_name in new_packages or req_name in self.packages: our_depends.append(req_name) else: - raise RuntimeError(f"Requirement {req_name} is not in this distribution.") + raise RuntimeError( + f"Requirement {req_name} is not in this distribution." + ) package.depends = our_depends while len(requirements_with_extras) != 0: extra_req = requirements_with_extras.pop() extra_package_name = canonicalize_name(r.name) - if extra_package_name in new_packages: - package = new_packages[extra_package_name] - our_depends = package.depends - wheel_file = new_package_wheels[package.name] - requirements = get_wheel_dependencies(wheel_file, package.name) - for extra in extra_req.extras: - for r in requirements: - req_marker = r.marker - req_name = canonicalize_name(r.name) - if req_marker is not None: - if req_marker.evaluate( - { - "sys_platform": "emscripten", - "platform_system": "Emscripten", - "extra": extra, - } - ): - if ( - req_name in new_packages - or req_name in self.packages - ): - our_depends.append(req_name) - if r.extras: - requirements_with_extras.append(r) - else: - raise RuntimeError(f"Requirement {req_name} is not in this distribution.") - package.depends = our_depends + if extra_package_name not in new_packages: + continue + package = new_packages[extra_package_name] + our_depends = package.depends + wheel_file = new_package_wheels[package.name] + requirements = get_wheel_dependencies(wheel_file, package.name) + for extra in extra_req.extras: + for r in requirements: + req_marker = r.marker + req_name = canonicalize_name(r.name) + if req_marker is None: + continue + if req_marker.evaluate( + { + "sys_platform": "emscripten", + "platform_system": "Emscripten", + "extra": extra, + } + ): + if req_name in new_packages or req_name in self.packages: + our_depends.append(req_name) + if r.extras: + requirements_with_extras.append(r) + else: + raise RuntimeError( + f"Requirement {req_name} is not in this distribution." + ) + package.depends = our_depends self.packages.update(new_packages) diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index e30cc4f..433c5d1 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -2,14 +2,12 @@ import logging import zipfile from collections import deque +from email.parser import BytesParser from pathlib import Path from zipfile import ZipFile -from email.parser import BytesParser -from packaging.utils import canonicalize_name as canonicalize_package_name -from packaging.utils import parse_wheel_filename from packaging.requirements import Requirement - +from packaging.utils import canonicalize_name as canonicalize_package_name logger = logging.getLogger(__name__) @@ -100,7 +98,7 @@ def get_wheel_dependencies(wheel_path: Path, pkg_name: str) -> list[Requirement] metadata_path = f"{dist_info_dir}/METADATA" p = BytesParser() headers = p.parse(wheel.open(metadata_path), headersonly=True) - requires : list[str] = headers.get_all("Requires-Dist", failobj=[]) + requires: list[str] = headers.get_all("Requires-Dist", failobj=[]) for r in requires: deps.append(Requirement(r)) return deps @@ -125,7 +123,8 @@ def get_wheel_dist_info_dir(wheel: ZipFile, pkg_name: str) -> str: if len(info_dirs) > 1: raise Exception( - f"multiple .dist-info directories found for {pkg_name}: {', '.join(info_dirs)}" + f"multiple .dist-info directories found for {pkg_name}:" + f"{', '.join(info_dirs)}" ) (info_dir,) = info_dirs From 4541c92cf91c7d247a0ebc5ef9156845d100984c Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 22 Sep 2023 14:26:12 +0100 Subject: [PATCH 07/22] fix tests --- pyodide_lock/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index 02a5339..39ee454 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -100,7 +100,7 @@ def check_wheel_filenames(self) -> None: f"{name}:\n - " + "\n - ".join(errs) for name, errs in errors.items() ) - raise RuntimeError(error_msg) + raise ValueError(error_msg) def add_wheels( # noqa: C901 self, From a5aac12dbe924ba2653276be8df5ebbb4df0339b Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 22 Sep 2023 15:41:41 +0100 Subject: [PATCH 08/22] test for add_wheels --- tests/test_add_wheels.py | 126 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 tests/test_add_wheels.py diff --git a/tests/test_add_wheels.py b/tests/test_add_wheels.py new file mode 100644 index 0000000..cc54eec --- /dev/null +++ b/tests/test_add_wheels.py @@ -0,0 +1,126 @@ +import json +from copy import deepcopy +from dataclasses import asdict, dataclass +from pathlib import Path +from tempfile import TemporaryDirectory + +import build +import pytest +from test_spec import LOCK_EXAMPLE + +from pyodide_lock import PyodideLockSpec + + +# build a wheel +def make_test_wheel( + dir: Path, + package_name: str, + deps: list[str] | None = None, + optional_deps: dict[str, list[str]] | None = None, + modules: list[str] | None = None, +): + package_dir = dir / package_name + package_dir.mkdir() + if modules is None: + modules = [package_name] + for m in modules: + (package_dir / f"{m}.py").write_text("") + toml = package_dir / "pyproject.toml" + if deps is None: + deps = [] + + all_deps = json.dumps(deps) + if optional_deps: + all_optional_deps = "[project.optional-dependencies]\n" + "\n".join( + [x + "=" + json.dumps(optional_deps[x]) for x in optional_deps.keys()] + ) + else: + all_optional_deps = "" + toml_text = f""" +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "{package_name}" +description = "{package_name} example package" +version = "1.0.0" +authors = [ + {{ name = "Bob Jones", email = "bobjones@nowhere.nowhere" }} +] +dependencies = { + all_deps +} + +{ all_optional_deps } + +""" + toml.write_text(toml_text) + builder = build.ProjectBuilder(package_dir) + return Path(builder.build("wheel", dir / "dist")) + + +@pytest.fixture(scope="module") +def test_wheel_list(): + @dataclass + class TestWheel: + package_name: str + modules: list[str] | None = None + deps: list[str] | None = None + optional_deps: dict[str, list[str]] | None = None + + test_wheels: list[TestWheel] = [ + TestWheel(package_name="py-one", modules=["one"]), + TestWheel(package_name="needs-one", deps=["py_one"]), + TestWheel(package_name="needs-one-opt", optional_deps={"with_one": ["py-one"]}), + TestWheel( + package_name="test-extra-dependencies", deps=["needs-one-opt[with_one]"] + ), + TestWheel(package_name="failure", deps=["two"]), + ] + + with TemporaryDirectory() as tmpdir: + path_temp = Path(tmpdir) + all_wheels = [] + for wheel_data in test_wheels: + all_wheels.append(make_test_wheel(path_temp, **asdict(wheel_data))) + yield all_wheels + + +def test_add_one(test_wheel_list): + lock_data = deepcopy(LOCK_EXAMPLE) + spec = PyodideLockSpec(**lock_data) + spec.add_wheels(test_wheel_list[0:1]) + # py_one only should get added + assert spec.packages["py-one"].imports == ["one"] + + +def test_add_simple_deps(test_wheel_list): + lock_data = deepcopy(LOCK_EXAMPLE) + spec = PyodideLockSpec(**lock_data) + spec.add_wheels(test_wheel_list[0:3]) + # py_one, needs_one and needs_one_opt should get added + assert "py-one" in spec.packages + assert "needs-one" in spec.packages + assert "needs-one-opt" in spec.packages + # needs one opt should not depend on py_one + assert spec.packages["needs-one-opt"].depends == [] + + +def test_add_deps_with_extras(test_wheel_list): + lock_data = deepcopy(LOCK_EXAMPLE) + spec = PyodideLockSpec(**lock_data) + spec.add_wheels(test_wheel_list[0:4]) + # py_one, needs_one, needs_one_opt and test_extra_dependencies should get added + # because of the extra dependency in test_extra_dependencies, + # needs_one_opt should now depend on one + assert "test-extra-dependencies" in spec.packages + assert spec.packages["needs-one-opt"].depends == ["py-one"] + + +def test_missing_dep(test_wheel_list): + lock_data = deepcopy(LOCK_EXAMPLE) + spec = PyodideLockSpec(**lock_data) + # this has a package with a missing dependency so should fail + with pytest.raises(RuntimeError): + spec.add_wheels(test_wheel_list[0:5]) From 5c20eaf77473c8d2b422e69b4905075cde73de5a Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 22 Sep 2023 19:14:27 +0100 Subject: [PATCH 09/22] add build to test dependencies --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e676285..76f9724 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dynamic = ["version"] [project.optional-dependencies] dev = [ - "pytest", "pytest-cov" + "pytest", "pytest-cov", "build" ] [project.urls] From 5dfe9e7d629693a7b797ab3dd5c27acecd8718dc Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 22 Sep 2023 19:19:54 +0100 Subject: [PATCH 10/22] add wheel to test deps --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 76f9724..15c2a5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dynamic = ["version"] [project.optional-dependencies] dev = [ - "pytest", "pytest-cov", "build" + "pytest", "pytest-cov", "build", "wheel" ] [project.urls] From ac33538e7518b68a651724b9c17e011dc767514f Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Thu, 28 Sep 2023 17:29:36 +0100 Subject: [PATCH 11/22] require build for tests --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e676285..9f3775c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dynamic = ["version"] [project.optional-dependencies] dev = [ - "pytest", "pytest-cov" + "pytest", "pytest-cov","build" ] [project.urls] From 84df621629718a031a72fb5712ca09ac1572a34a Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 29 Sep 2023 12:57:42 +0100 Subject: [PATCH 12/22] bugfix to _get_marker_environment --- pyodide_lock/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index c28f5d9..5de83e8 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -113,6 +113,7 @@ def _get_marker_environment(platform: str, version: str, arch: str, python: str) """ try: + exec("import pyodide") from packaging.markers import default_environment return default_environment() From 07a73e1360474a1072addbfdeb45799d8da0a285 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 29 Sep 2023 13:54:27 +0100 Subject: [PATCH 13/22] fully formed cli --- pyodide_lock/cli/lockfile.py | 48 +++++++++++++++++++++++++++++------- pyodide_lock/spec.py | 2 +- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/pyodide_lock/cli/lockfile.py b/pyodide_lock/cli/lockfile.py index 3c5120a..ec69c7c 100644 --- a/pyodide_lock/cli/lockfile.py +++ b/pyodide_lock/cli/lockfile.py @@ -1,5 +1,4 @@ from pathlib import Path -from typing import Annotated import typer @@ -10,14 +9,45 @@ @main.command() def add_wheels( - wheels: Annotated[ - list[Path], - typer.Argument(help="list of wheels to add to the lockfile", default="[]"), - ], - in_lockfile: Path = Path("pyodide-lock.json"), - out_lockfile: Path = Path("pyodide-lock-new.json"), + wheels: list[Path], + ignore_missing_dependencies: bool = typer.Option( + help="If this is true, dependencies " + "which are not in the original lockfile or " + "the added wheels will be added to the lockfile. " + "Warning: This will allow a broken lockfile to " + "be created.", + default=False, + ), + in_lockfile: Path = typer.Option( + help="Source lockfile (input)", default=Path("pyodide-lock.json") + ), + out_lockfile: Path = typer.Option( + help="Updated lockfile (output)", default=Path("pyodide-lock-new.json") + ), + base_path: Path = typer.Option( + help="Base path for wheels - wheel file " + "names will be created relative to this path.", + default=None, + ), + wheel_url: str = typer.Option( + help="Base url which will be appended to the wheel location." + "Use this if you are hosting these wheels on a different " + "server to core pyodide packages", + default="", + ), ): - """Add a set of wheels to an existing pyodide-lock.json""" + """Add a set of wheels to an existing pyodide-lock.json. + + Each package in the wheel will be added to the output lockfile, + including resolution of dependencies in the lock file. By default + this will fail if a dependency isn't available in either the + existing lock file, or in the set of new wheels. + """ sp = PyodideLockSpec.from_json(in_lockfile) - sp.add_wheels(wheels) + sp.add_wheels( + wheels, + base_path=base_path, + base_url=wheel_url, + ignore_missing_dependencies=ignore_missing_dependencies, + ) sp.to_json(out_lockfile) diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index 48f8e45..d560762 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -275,7 +275,7 @@ def _fix_extra_dep( marker_environment = _get_marker_environment(**self.info.dict()) extra_package_name = canonicalize_name(extra_req.name) if extra_package_name not in new_packages: - return + return [] package = new_packages[extra_package_name] our_depends = package.depends wheel_file = package.file_name From 1bc04eef0175afbe6f2f2856c3db300183ae2af2 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 29 Sep 2023 13:55:35 +0100 Subject: [PATCH 14/22] improved docs --- pyodide_lock/cli/lockfile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyodide_lock/cli/lockfile.py b/pyodide_lock/cli/lockfile.py index ec69c7c..e16b32b 100644 --- a/pyodide_lock/cli/lockfile.py +++ b/pyodide_lock/cli/lockfile.py @@ -36,12 +36,14 @@ def add_wheels( default="", ), ): - """Add a set of wheels to an existing pyodide-lock.json. + """Add a set of package wheels to an existing pyodide-lock.json and + write it out to pyodide-lock-new.json Each package in the wheel will be added to the output lockfile, including resolution of dependencies in the lock file. By default this will fail if a dependency isn't available in either the existing lock file, or in the set of new wheels. + """ sp = PyodideLockSpec.from_json(in_lockfile) sp.add_wheels( From 44a80bb43782934b426d7466717f451f4564d402 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 29 Sep 2023 14:11:06 +0100 Subject: [PATCH 15/22] add typer to dev dependencies --- pyodide_lock/cli/lockfile.py | 98 +++++++++++++++++++----------------- pyproject.toml | 4 ++ 2 files changed, 55 insertions(+), 47 deletions(-) diff --git a/pyodide_lock/cli/lockfile.py b/pyodide_lock/cli/lockfile.py index e16b32b..771b51a 100644 --- a/pyodide_lock/cli/lockfile.py +++ b/pyodide_lock/cli/lockfile.py @@ -1,55 +1,59 @@ from pathlib import Path -import typer +try: + import typer -from ..spec import PyodideLockSpec + from ..spec import PyodideLockSpec -main = typer.Typer() + main = typer.Typer() + @main.command() + def add_wheels( + wheels: list[Path], + ignore_missing_dependencies: bool = typer.Option( + help="If this is true, dependencies " + "which are not in the original lockfile or " + "the added wheels will be added to the lockfile. " + "Warning: This will allow a broken lockfile to " + "be created.", + default=False, + ), + in_lockfile: Path = typer.Option( + help="Source lockfile (input)", default=Path("pyodide-lock.json") + ), + out_lockfile: Path = typer.Option( + help="Updated lockfile (output)", default=Path("pyodide-lock-new.json") + ), + base_path: Path = typer.Option( + help="Base path for wheels - wheel file " + "names will be created relative to this path.", + default=None, + ), + wheel_url: str = typer.Option( + help="Base url which will be appended to the wheel location." + "Use this if you are hosting these wheels on a different " + "server to core pyodide packages", + default="", + ), + ): + """Add a set of package wheels to an existing pyodide-lock.json and + write it out to pyodide-lock-new.json -@main.command() -def add_wheels( - wheels: list[Path], - ignore_missing_dependencies: bool = typer.Option( - help="If this is true, dependencies " - "which are not in the original lockfile or " - "the added wheels will be added to the lockfile. " - "Warning: This will allow a broken lockfile to " - "be created.", - default=False, - ), - in_lockfile: Path = typer.Option( - help="Source lockfile (input)", default=Path("pyodide-lock.json") - ), - out_lockfile: Path = typer.Option( - help="Updated lockfile (output)", default=Path("pyodide-lock-new.json") - ), - base_path: Path = typer.Option( - help="Base path for wheels - wheel file " - "names will be created relative to this path.", - default=None, - ), - wheel_url: str = typer.Option( - help="Base url which will be appended to the wheel location." - "Use this if you are hosting these wheels on a different " - "server to core pyodide packages", - default="", - ), -): - """Add a set of package wheels to an existing pyodide-lock.json and - write it out to pyodide-lock-new.json + Each package in the wheel will be added to the output lockfile, + including resolution of dependencies in the lock file. By default + this will fail if a dependency isn't available in either the + existing lock file, or in the set of new wheels. - Each package in the wheel will be added to the output lockfile, - including resolution of dependencies in the lock file. By default - this will fail if a dependency isn't available in either the - existing lock file, or in the set of new wheels. + """ + sp = PyodideLockSpec.from_json(in_lockfile) + sp.add_wheels( + wheels, + base_path=base_path, + base_url=wheel_url, + ignore_missing_dependencies=ignore_missing_dependencies, + ) + sp.to_json(out_lockfile) - """ - sp = PyodideLockSpec.from_json(in_lockfile) - sp.add_wheels( - wheels, - base_path=base_path, - base_url=wheel_url, - ignore_missing_dependencies=ignore_missing_dependencies, - ) - sp.to_json(out_lockfile) +except ImportError: + pass + # no typer = no cli diff --git a/pyproject.toml b/pyproject.toml index 7604549..7df91d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,9 @@ classifiers = [ dynamic = ["version"] [project.optional-dependencies] +cli = [ + "typer", +] wheel = [ "pkginfo", "packaging", @@ -30,6 +33,7 @@ dev = [ "pytest", "pytest-cov", "build", + "typer", # from wheel "pkginfo", "packaging", From e0fb69cc4e28d8e66842d1442d64fb2bcf431c31 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Mon, 2 Oct 2023 13:19:30 +0100 Subject: [PATCH 16/22] updates for review --- pyodide_lock/{cli/lockfile.py => cli.py} | 2 +- pyodide_lock/cli/__init__.py | 0 pyodide_lock/spec.py | 6 ++++-- pyproject.toml | 2 +- tests/test_wheel.py | 3 ++- 5 files changed, 8 insertions(+), 5 deletions(-) rename pyodide_lock/{cli/lockfile.py => cli.py} (98%) delete mode 100644 pyodide_lock/cli/__init__.py diff --git a/pyodide_lock/cli/lockfile.py b/pyodide_lock/cli.py similarity index 98% rename from pyodide_lock/cli/lockfile.py rename to pyodide_lock/cli.py index 771b51a..edaf485 100644 --- a/pyodide_lock/cli/lockfile.py +++ b/pyodide_lock/cli.py @@ -3,7 +3,7 @@ try: import typer - from ..spec import PyodideLockSpec + from .spec import PyodideLockSpec main = typer.Typer() diff --git a/pyodide_lock/cli/__init__.py b/pyodide_lock/cli/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index d560762..317694f 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Literal -from pydantic import BaseModel, Extra +from pydantic import BaseModel, Extra, Field if TYPE_CHECKING: from packaging.requirements import Requirement @@ -30,7 +30,9 @@ class Config: class PackageSpec(BaseModel): name: str version: str - file_name: str + file_name: str = Field( + description="Path (or URL) to wheel.", format="uri-reference" + ) install_dir: str sha256: str = "" package_type: Literal[ diff --git a/pyproject.toml b/pyproject.toml index 7df91d3..888e63d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,4 +83,4 @@ addopts = ''' --doctest-modules''' [project.entry-points."pyodide.cli"] -lockfile = "pyodide_lock.cli.lockfile:main" +lockfile = "pyodide_lock.cli:main" diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 6496e7f..7395874 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -7,11 +7,12 @@ import build import pytest -from test_spec import LOCK_EXAMPLE from pyodide_lock import PackageSpec, PyodideLockSpec from pyodide_lock.utils import _generate_package_hash, _get_marker_environment +from .test_spec import LOCK_EXAMPLE + # we test if our own wheel imports nicely # so check if it is built in /dist, or else skip that test HERE = Path(__file__).parent From 8ed44edb625cab708483f7dfd9b3916f761d6fc4 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Mon, 2 Oct 2023 13:35:35 +0100 Subject: [PATCH 17/22] moved anything that modifies spec into utils --- pyodide_lock/spec.py | 225 +---------------------------------------- pyodide_lock/utils.py | 226 +++++++++++++++++++++++++++++++++++++++++- tests/test_spec.py | 5 +- tests/test_wheel.py | 33 +++--- 4 files changed, 248 insertions(+), 241 deletions(-) diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index 317694f..b287174 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -1,20 +1,11 @@ import json -import re from pathlib import Path from typing import TYPE_CHECKING, Literal from pydantic import BaseModel, Extra, Field if TYPE_CHECKING: - from packaging.requirements import Requirement - -from .utils import ( - _generate_package_hash, - _get_marker_environment, - _wheel_depends, - _wheel_metadata, - parse_top_level_import_name, -) + pass class InfoSpec(BaseModel): @@ -47,80 +38,6 @@ class PackageSpec(BaseModel): class Config: extra = Extra.forbid - @classmethod - def _from_wheel(cls, path: Path, info: InfoSpec) -> "PackageSpec": - """Build a package spec from an on-disk wheel. - - This is internal, because to reliably handle dependencies, we need: - 1) To have access to all the wheels being added at once (to handle extras) - 2) To know whether dependencies are available in the combined lockfile. - 3) To fix up wheel urls and paths consistently - - This is called by PyodideLockSpec.add_wheels below. - """ - from packaging.utils import ( - InvalidWheelFilename, - canonicalize_name, - parse_wheel_filename, - ) - from packaging.version import InvalidVersion - from packaging.version import parse as version_parse - - path = path.absolute() - # throw an error if this is an incompatible wheel - target_python = version_parse(info.python) - target_platform = info.platform + "_" + info.arch - try: - (name, version, build_number, tags) = parse_wheel_filename(str(path.name)) - except (InvalidWheelFilename, InvalidVersion) as e: - raise RuntimeError(f"Wheel filename {path.name} is not valid") from e - python_binary_abi = f"cp{target_python.major}{target_python.minor}" - tags = list(tags) - tag_match = False - for t in tags: - # abi should be - if ( - t.abi == python_binary_abi - and t.interpreter == python_binary_abi - and t.platform == target_platform - ): - tag_match = True - elif t.abi == "none" and t.platform == "any": - match = re.match(rf"py{target_python.major}(\d*)", t.interpreter) - if match: - subver = match.group(1) - if len(subver) == 0 or int(subver) <= target_python.minor: - tag_match = True - if not tag_match: - raise RuntimeError( - f"Package tags {tags} don't match Python version in lockfile:" - f"Lockfile python {target_python.major}.{target_python.minor}" - f"on platform {target_platform} ({python_binary_abi})" - ) - metadata = _wheel_metadata(path) - - if not metadata: - raise RuntimeError(f"Could not parse wheel metadata from {path.name}") - - # returns a draft PackageSpec with: - # 1) absolute path to wheel, - # 2) empty dependency list - return PackageSpec( - name=canonicalize_name(metadata.name), - version=metadata.version, - file_name=str(path), - sha256=_generate_package_hash(path), - package_type="package", - install_dir="site", - imports=parse_top_level_import_name(path), - depends=[], - ) - - def update_sha256(self, path: Path) -> "PackageSpec": - """Update the sha256 hash for a package.""" - self.sha256 = _generate_package_hash(path) - return self - class PyodideLockSpec(BaseModel): """A specification for the pyodide-lock.json file.""" @@ -175,143 +92,3 @@ def check_wheel_filenames(self) -> None: for name, errs in errors.items() ) raise ValueError(error_msg) - - def add_wheels( - self, - wheel_files: list[Path], - base_path: Path | None = None, - base_url: str = "", - ignore_missing_dependencies: bool = False, - ) -> None: - """Add a list of wheel files to this pyodide-lock.json - - Args: - wheel_files (list[Path]): A list of wheel files to import. - base_path (Path | None, optional): - Filenames are stored relative to this base path. By default the - filename is stored relative to the path of the first wheel file - in the list. - - base_url (str, optional): - The base URL stored in the pyodide-lock.json. By default this - is empty which means that wheels must be stored in the same folder - as the core pyodide packages you are using. If you want to store - your custom wheels somewhere else, set this base_url to point to it. - """ - if len(wheel_files) <= 0: - return - wheel_files = [f.resolve() for f in wheel_files] - if base_path is None: - base_path = wheel_files[0].parent - else: - base_path = base_path.resolve() - - new_packages = {} - for f in wheel_files: - spec = PackageSpec._from_wheel(f, info=self.info) - - new_packages[spec.name] = spec - - self._fix_new_package_deps(new_packages, ignore_missing_dependencies) - self._set_package_paths(new_packages, base_path, base_url) - self.packages |= new_packages - - def _fix_new_package_deps( - self, new_packages: dict[str, PackageSpec], ignore_missing_dependencies: bool - ): - # now fix up the dependencies for each of our new packages - # n.b. this assumes existing packages have correct dependencies, - # which is probably a good assumption. - from packaging.utils import canonicalize_name - - requirements_with_extras = [] - marker_environment = _get_marker_environment(**self.info.dict()) - for package in new_packages.values(): - # add any requirements to the list of packages - our_depends = [] - wheel_file = package.file_name - metadata = _wheel_metadata(wheel_file) - requirements = _wheel_depends(metadata) - for r in requirements: - req_marker = r.marker - req_name = canonicalize_name(r.name) - if req_marker is not None: - if not req_marker.evaluate(marker_environment): - # not used in pyodide / emscripten - # or optional requirement - continue - if r.extras: - # this requirement has some extras, we need to check - # that the required package depends on these extras also. - requirements_with_extras.append(r) - if req_name in new_packages or req_name in self.packages: - our_depends.append(req_name) - elif ignore_missing_dependencies: - our_depends.append(req_name) - else: - raise RuntimeError( - f"Requirement {req_name} from {r} is not in this distribution." - ) - package.depends = our_depends - while len(requirements_with_extras) != 0: - extra_req = requirements_with_extras.pop() - requirements_with_extras.extend( - self._fix_extra_dep( - extra_req, new_packages, ignore_missing_dependencies - ) - ) - - # When requirements have extras, we need to make sure that the - # required package includes the dependencies for that extra. - # This is because extras aren't supported in pyodide-lock - def _fix_extra_dep( - self, - extra_req: "Requirement", - new_packages: dict[str, PackageSpec], - ignore_missing_dependencies: bool, - ): - from packaging.utils import canonicalize_name - - requirements_with_extras = [] - - marker_environment = _get_marker_environment(**self.info.dict()) - extra_package_name = canonicalize_name(extra_req.name) - if extra_package_name not in new_packages: - return [] - package = new_packages[extra_package_name] - our_depends = package.depends - wheel_file = package.file_name - metadata = _wheel_metadata(wheel_file) - requirements = _wheel_depends(metadata) - for extra in extra_req.extras: - this_marker_env = marker_environment.copy() - this_marker_env["extra"] = extra - - for r in requirements: - req_marker = r.marker - req_name = canonicalize_name(r.name) - if req_name not in our_depends: - if req_marker is None: - # no marker - this will have been processed above - continue - if req_marker.evaluate(this_marker_env): - if req_name in new_packages or req_name in self.packages: - our_depends.append(req_name) - if r.extras: - requirements_with_extras.append(r) - elif ignore_missing_dependencies: - our_depends.append(req_name) - else: - raise RuntimeError( - f"Requirement {req_name} is not in this distribution." - ) - package.depends = our_depends - return requirements_with_extras - - def _set_package_paths( - self, new_packages: dict[str, PackageSpec], base_path: Path, base_url: str - ): - for p in new_packages.values(): - current_path = Path(p.file_name) - relative_path = current_path.relative_to(base_path) - p.file_name = base_url + str(relative_path) diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index 5de83e8..d4a7db0 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -5,7 +5,9 @@ from collections import deque from functools import cache from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING # + +from .spec import InfoSpec, PackageSpec, PyodideLockSpec if TYPE_CHECKING: from packaging.requirements import Requirement @@ -153,3 +155,225 @@ def _wheel_depends(metadata: "Distribution") -> list["Requirement"]: depends.append(req) return depends + + +def add_wheels_to_spec( + lock_spec: PyodideLockSpec, + wheel_files: list[Path], + base_path: Path | None = None, + base_url: str = "", + ignore_missing_dependencies: bool = False, +) -> None: + """Add a list of wheel files to this pyodide-lock.json + + Args: + wheel_files (list[Path]): A list of wheel files to import. + base_path (Path | None, optional): + Filenames are stored relative to this base path. By default the + filename is stored relative to the path of the first wheel file + in the list. + + base_url (str, optional): + The base URL stored in the pyodide-lock.json. By default this + is empty which means that wheels must be stored in the same folder + as the core pyodide packages you are using. If you want to store + your custom wheels somewhere else, set this base_url to point to it. + """ + if len(wheel_files) <= 0: + return + wheel_files = [f.resolve() for f in wheel_files] + if base_path is None: + base_path = wheel_files[0].parent + else: + base_path = base_path.resolve() + + new_packages = {} + for f in wheel_files: + spec = package_spec_from_wheel(f, info=lock_spec.info) + + new_packages[spec.name] = spec + + _fix_new_package_deps(lock_spec, new_packages, ignore_missing_dependencies) + _set_package_paths(new_packages, base_path, base_url) + lock_spec.packages |= new_packages + + +def _fix_new_package_deps( + lock_spec: PyodideLockSpec, + new_packages: dict[str, PackageSpec], + ignore_missing_dependencies: bool, +): + # now fix up the dependencies for each of our new packages + # n.b. this assumes existing packages have correct dependencies, + # which is probably a good assumption. + from packaging.utils import canonicalize_name + + requirements_with_extras = [] + marker_environment = _get_marker_environment(**lock_spec.info.dict()) + for package in new_packages.values(): + # add any requirements to the list of packages + our_depends = [] + wheel_file = package.file_name + metadata = _wheel_metadata(wheel_file) + requirements = _wheel_depends(metadata) + for r in requirements: + req_marker = r.marker + req_name = canonicalize_name(r.name) + if req_marker is not None: + if not req_marker.evaluate(marker_environment): + # not used in pyodide / emscripten + # or optional requirement + continue + if r.extras: + # this requirement has some extras, we need to check + # that the required package depends on these extras also. + requirements_with_extras.append(r) + if req_name in new_packages or req_name in lock_spec.packages: + our_depends.append(req_name) + elif ignore_missing_dependencies: + our_depends.append(req_name) + else: + raise RuntimeError( + f"Requirement {req_name} from {r} is not in this distribution." + ) + package.depends = our_depends + while len(requirements_with_extras) != 0: + extra_req = requirements_with_extras.pop() + requirements_with_extras.extend( + _fix_extra_dep( + lock_spec, extra_req, new_packages, ignore_missing_dependencies + ) + ) + + +# When requirements have extras, we need to make sure that the +# required package includes the dependencies for that extra. +# This is because extras aren't supported in pyodide-lock +def _fix_extra_dep( + lock_spec: PyodideLockSpec, + extra_req: "Requirement", + new_packages: dict[str, PackageSpec], + ignore_missing_dependencies: bool, +): + from packaging.utils import canonicalize_name + + requirements_with_extras = [] + + marker_environment = _get_marker_environment(**lock_spec.info.dict()) + extra_package_name = canonicalize_name(extra_req.name) + if extra_package_name not in new_packages: + return [] + package = new_packages[extra_package_name] + our_depends = package.depends + wheel_file = package.file_name + metadata = _wheel_metadata(wheel_file) + requirements = _wheel_depends(metadata) + for extra in extra_req.extras: + this_marker_env = marker_environment.copy() + this_marker_env["extra"] = extra + + for r in requirements: + req_marker = r.marker + req_name = canonicalize_name(r.name) + if req_name not in our_depends: + if req_marker is None: + # no marker - this will have been processed above + continue + if req_marker.evaluate(this_marker_env): + if req_name in new_packages or req_name in lock_spec.packages: + our_depends.append(req_name) + if r.extras: + requirements_with_extras.append(r) + elif ignore_missing_dependencies: + our_depends.append(req_name) + else: + raise RuntimeError( + f"Requirement {req_name} is not in this distribution." + ) + package.depends = our_depends + return requirements_with_extras + + +def _set_package_paths( + new_packages: dict[str, PackageSpec], base_path: Path, base_url: str +): + for p in new_packages.values(): + current_path = Path(p.file_name) + relative_path = current_path.relative_to(base_path) + p.file_name = base_url + str(relative_path) + + +def package_spec_from_wheel(path: Path, info: InfoSpec) -> "PackageSpec": + """Build a package spec from an on-disk wheel. + + Warning - to reliably handle dependencies, we need: + 1) To have access to all the wheels being added at once (to handle extras) + 2) To know whether dependencies are available in the combined lockfile. + 3) To fix up wheel urls and paths consistently + + This is called by add_wheels_to_spec + """ + from packaging.utils import ( + InvalidWheelFilename, + canonicalize_name, + parse_wheel_filename, + ) + from packaging.version import InvalidVersion + from packaging.version import parse as version_parse + + path = path.absolute() + # throw an error if this is an incompatible wheel + target_python = version_parse(info.python) + target_platform = info.platform + "_" + info.arch + try: + (name, version, build_number, tags) = parse_wheel_filename(str(path.name)) + except (InvalidWheelFilename, InvalidVersion) as e: + raise RuntimeError(f"Wheel filename {path.name} is not valid") from e + python_binary_abi = f"cp{target_python.major}{target_python.minor}" + tags = list(tags) + + tag_match = False + for t in tags: + # abi should be + if ( + t.abi == python_binary_abi + and t.interpreter == python_binary_abi + and t.platform == target_platform + ): + tag_match = True + elif t.abi == "none" and t.platform == "any": + match = re.match(rf"py{target_python.major}(\d*)", t.interpreter) + if match: + subver = match.group(1) + if len(subver) == 0 or int(subver) <= target_python.minor: + tag_match = True + if not tag_match: + raise RuntimeError( + f"Package tags {tags} don't match Python version in lockfile:" + f"Lockfile python {target_python.major}.{target_python.minor}" + f"on platform {target_platform} ({python_binary_abi})" + ) + metadata = _wheel_metadata(path) + + if not metadata: + raise RuntimeError(f"Could not parse wheel metadata from {path.name}") + + # returns a draft PackageSpec with: + # 1) absolute path to wheel, + # 2) empty dependency list + return PackageSpec( + name=canonicalize_name(metadata.name), + version=metadata.version, + file_name=str(path), + sha256=_generate_package_hash(path), + package_type="package", + install_dir="site", + imports=parse_top_level_import_name(path), + depends=[], + ) + + +def update_package_sha256(spec: PackageSpec, path: Path) -> "PackageSpec": + """Update the sha256 hash for a package.""" + spec.sha256 = _generate_package_hash(path) + return spec diff --git a/tests/test_spec.py b/tests/test_spec.py index 7d29ae2..5613e6a 100644 --- a/tests/test_spec.py +++ b/tests/test_spec.py @@ -7,6 +7,7 @@ from pyodide_lock import PyodideLockSpec from pyodide_lock.spec import InfoSpec, PackageSpec +from pyodide_lock.utils import update_package_sha256 DATA_DIR = Path(__file__).parent / "data" @@ -98,13 +99,13 @@ def test_to_json_indent(tmp_path): def test_update_sha256(monkeypatch): - monkeypatch.setattr("pyodide_lock.spec._generate_package_hash", lambda x: "abcd") + monkeypatch.setattr("pyodide_lock.utils._generate_package_hash", lambda x: "abcd") lock_data = deepcopy(LOCK_EXAMPLE) lock_data["packages"]["numpy"]["sha256"] = "0" # type: ignore[index] spec = PyodideLockSpec(**lock_data) assert spec.packages["numpy"].sha256 == "0" - spec.packages["numpy"].update_sha256(Path("/some/path")) + update_package_sha256(spec.packages["numpy"], Path("/some/path")) assert spec.packages["numpy"].sha256 == "abcd" diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 7395874..08464c6 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -7,11 +7,14 @@ import build import pytest +from test_spec import LOCK_EXAMPLE from pyodide_lock import PackageSpec, PyodideLockSpec -from pyodide_lock.utils import _generate_package_hash, _get_marker_environment - -from .test_spec import LOCK_EXAMPLE +from pyodide_lock.utils import ( + _generate_package_hash, + _get_marker_environment, + add_wheels_to_spec, +) # we test if our own wheel imports nicely # so check if it is built in /dist, or else skip that test @@ -130,7 +133,8 @@ class TestWheel: def test_add_one(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels(test_wheel_list[0:1]) + + add_wheels_to_spec(spec, test_wheel_list[0:1]) # py_one only should get added assert spec.packages["py-one"].imports == ["one"] @@ -138,7 +142,7 @@ def test_add_one(test_wheel_list): def test_add_simple_deps(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels(test_wheel_list[0:3]) + add_wheels_to_spec(spec, test_wheel_list[0:3]) # py_one, needs_one and needs_one_opt should get added assert "py-one" in spec.packages assert "needs-one" in spec.packages @@ -152,7 +156,7 @@ def test_add_simple_deps(test_wheel_list): def test_add_deps_with_extras(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels(test_wheel_list[0:4]) + add_wheels_to_spec(spec, test_wheel_list[0:4]) # py_one, needs_one, needs_one_opt and test_extra_dependencies should get added # because of the extra dependency in test_extra_dependencies, # needs_one_opt should now depend on one @@ -165,13 +169,13 @@ def test_missing_dep(test_wheel_list): spec = PyodideLockSpec(**lock_data) # this has a package with a missing dependency so should fail with pytest.raises(RuntimeError): - spec.add_wheels(test_wheel_list[0:5]) + add_wheels_to_spec(spec, test_wheel_list[0:5]) def test_path_rewriting(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels(test_wheel_list[0:3], base_url="http://www.nowhere.org/") + add_wheels_to_spec(spec, test_wheel_list[0:3], base_url="http://www.nowhere.org/") # py_one, needs_one and needs_one_opt should get added assert "py-one" in spec.packages assert "needs-one" in spec.packages @@ -181,7 +185,8 @@ def test_path_rewriting(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) # this should add the base path "dist" to the file name - spec.add_wheels( + add_wheels_to_spec( + spec, test_wheel_list[0:3], base_url="http://www.nowhere.org/", base_path=test_wheel_list[0].parent.parent, @@ -199,7 +204,7 @@ def test_path_rewriting(test_wheel_list): def test_markers_not_needed(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels(test_wheel_list[5:6]) + add_wheels_to_spec(spec, test_wheel_list[5:6]) assert spec.packages["markers-not-needed-test"].depends == [] @@ -208,7 +213,7 @@ def test_markers_not_needed(test_wheel_list): def test_markers_needed(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels(test_wheel_list[6:7], ignore_missing_dependencies=True) + add_wheels_to_spec(spec, test_wheel_list[6:7], ignore_missing_dependencies=True) assert len(spec.packages["markers-needed-test"].depends) == len( MARKER_EXAMPLES_NEEDED ) @@ -219,7 +224,7 @@ def test_self_wheel(): assert WHEEL is not None lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels([WHEEL], ignore_missing_dependencies=True) + add_wheels_to_spec(spec, [WHEEL], ignore_missing_dependencies=True) expected = PackageSpec( name="pyodide-lock", @@ -245,7 +250,7 @@ def test_not_wheel(tmp_path): whlzip.writestr("README.md", data="Not a wheel") with pytest.raises(RuntimeError, match="metadata"): - spec.add_wheels([wheel]) + add_wheels_to_spec(spec, [wheel]) @pytest.mark.parametrize( @@ -262,4 +267,4 @@ def test_bad_names(tmp_path, bad_name): with zipfile.ZipFile(wheel, "w") as whlzip: whlzip.writestr("README.md", data="Not a wheel") with pytest.raises(RuntimeError, match="Wheel filename"): - spec.add_wheels([wheel]) + add_wheels_to_spec(spec, [wheel]) From c4f0985c4b859a35858aa9a4cb172fb45e04c3f7 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Mon, 2 Oct 2023 13:37:48 +0100 Subject: [PATCH 18/22] updated cli for changes in add_wheels_to_spec --- pyodide_lock/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyodide_lock/cli.py b/pyodide_lock/cli.py index edaf485..9fb3fc6 100644 --- a/pyodide_lock/cli.py +++ b/pyodide_lock/cli.py @@ -4,6 +4,7 @@ import typer from .spec import PyodideLockSpec + from .utils import add_wheels_to_spec main = typer.Typer() @@ -46,7 +47,8 @@ def add_wheels( """ sp = PyodideLockSpec.from_json(in_lockfile) - sp.add_wheels( + add_wheels_to_spec( + sp, wheels, base_path=base_path, base_url=wheel_url, From 462ecb2267b524108b969646e7714190f9c7f4ac Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Thu, 5 Oct 2023 12:10:29 +0100 Subject: [PATCH 19/22] moved shared test stuff to conftest.py --- tests/conftest.py | 160 ++++++++++++++++++++++++++++++ tests/test_spec.py | 61 ++++-------- tests/test_wheel.py | 234 ++++++++++---------------------------------- 3 files changed, 231 insertions(+), 224 deletions(-) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6c04f7e --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,160 @@ +import json +from copy import deepcopy +from dataclasses import asdict, dataclass +from pathlib import Path +from tempfile import TemporaryDirectory + +import build +import pytest + +from pyodide_lock import PyodideLockSpec +from pyodide_lock.utils import _get_marker_environment + +LOCK_EXAMPLE = { + "info": { + "arch": "wasm32", + "platform": "emscripten_3_1_39", + "version": "0.24.0.dev0", + "python": "3.11.3", + }, + "packages": { + "numpy": { + "name": "numpy", + "version": "1.24.3", + "file_name": "numpy-1.24.3-cp311-cp311-emscripten_3_1_39_wasm32.whl", + "install_dir": "site", + "sha256": ( + "513af43ffb1f7d507c8d879c9f7e5" "d6c789ad21b6a67e5bca1d7cfb86bf8640f" + ), + "imports": ["numpy"], + "depends": [], + } + }, +} + +# marker environment for testing +_ENV = _get_marker_environment(**LOCK_EXAMPLE["info"]) # type:ignore[arg-type] +# marker environment for testing, filtered only to numerical values +_ENV_NUM = {k: v for k, v in _ENV.items() if v[0] in "0123456789"} + +MARKER_EXAMPLES_NOT_NEEDED = ( + [ + 'requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < "2.7"', + 'argparse;python_version<"2.7"', + ] + + [f'Not.expected ; {k} != "{v}"' for k, v in _ENV.items()] + + [f'Not.expected ; {k} > "{v}"' for k, v in _ENV_NUM.items()] +) + + +MARKER_EXAMPLES_NEEDED = ( + [ + 'a;python_version>="3.5"', + 'b;sys_platform=="emscripten"', + ] + + [f'c_{k}; {k} == "{v}"' for k, v in _ENV.items()] + + [f'd_{k} ; {k} <= "{v}"' for k, v in _ENV_NUM.items()] +) + + +@pytest.fixture +def marker_examples_needed(): + return MARKER_EXAMPLES_NEEDED + + +@pytest.fixture +def marker_examples_not_needed(): + return MARKER_EXAMPLES_NOT_NEEDED + + +@pytest.fixture +def example_lock_data(): + return deepcopy(LOCK_EXAMPLE) + + +@pytest.fixture +def example_lock_spec(): + return PyodideLockSpec(**deepcopy(LOCK_EXAMPLE)) + + +# build a wheel +def make_test_wheel( + dir: Path, + package_name: str, + deps: list[str] | None = None, + optional_deps: dict[str, list[str]] | None = None, + modules: list[str] | None = None, +): + package_dir = dir / package_name + package_dir.mkdir() + if modules is None: + modules = [package_name] + for m in modules: + (package_dir / f"{m}.py").write_text("") + toml = package_dir / "pyproject.toml" + if deps is None: + deps = [] + + all_deps = json.dumps(deps) + if optional_deps: + all_optional_deps = "[project.optional-dependencies]\n" + "\n".join( + [x + "=" + json.dumps(optional_deps[x]) for x in optional_deps.keys()] + ) + else: + all_optional_deps = "" + toml_text = f""" +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "{package_name}" +description = "{package_name} example package" +version = "1.0.0" +authors = [ + {{ name = "Bob Jones", email = "bobjones@nowhere.nowhere" }} +] +dependencies = { + all_deps +} + +{ all_optional_deps } + +""" + toml.write_text(toml_text) + builder = build.ProjectBuilder(package_dir) + return Path(builder.build("wheel", dir / "dist")) + + +@pytest.fixture(scope="module") +def test_wheel_list(): + @dataclass + class TestWheel: + package_name: str + modules: list[str] | None = None + deps: list[str] | None = None + optional_deps: dict[str, list[str]] | None = None + + # a set of test wheels - note that names are non-canonicalized + # deliberately to test this + test_wheels: list[TestWheel] = [ + TestWheel(package_name="py-one", modules=["one"]), + TestWheel(package_name="NEeds-one", deps=["py_one"]), + TestWheel(package_name="nEEds-one-opt", optional_deps={"with_one": ["py_One"]}), + TestWheel( + package_name="test-extra_dependencies", deps=["needs-one-opt[with_one]"] + ), + TestWheel(package_name="failure", deps=["two"]), + TestWheel( + package_name="markers_not_needed_test", deps=MARKER_EXAMPLES_NOT_NEEDED + ), + TestWheel(package_name="markers_needed_test", deps=MARKER_EXAMPLES_NEEDED), + ] + + with TemporaryDirectory() as tmpdir: + path_temp = Path(tmpdir) + path_temp.mkdir(exist_ok=True) + all_wheels = [] + for wheel_data in test_wheels: + all_wheels.append(make_test_wheel(path_temp, **asdict(wheel_data))) + yield all_wheels diff --git a/tests/test_spec.py b/tests/test_spec.py index 5613e6a..0a78576 100644 --- a/tests/test_spec.py +++ b/tests/test_spec.py @@ -11,28 +11,6 @@ DATA_DIR = Path(__file__).parent / "data" -LOCK_EXAMPLE = { - "info": { - "arch": "wasm32", - "platform": "emscripten_3_1_39", - "version": "0.24.0.dev0", - "python": "3.11.3", - }, - "packages": { - "numpy": { - "name": "numpy", - "version": "1.24.3", - "file_name": "numpy-1.24.3-cp311-cp311-emscripten_3_1_39_wasm32.whl", - "install_dir": "site", - "sha256": ( - "513af43ffb1f7d507c8d879c9f7e5" "d6c789ad21b6a67e5bca1d7cfb86bf8640f" - ), - "imports": ["numpy"], - "depends": [], - } - }, -} - @pytest.mark.parametrize("pyodide_version", ["0.22.1", "0.23.3"]) def test_lock_spec_parsing(pyodide_version, tmp_path): @@ -55,14 +33,12 @@ def test_lock_spec_parsing(pyodide_version, tmp_path): assert spec.packages[key] == spec2.packages[key] -def test_check_wheel_filenames(): - lock_data = deepcopy(LOCK_EXAMPLE) - - spec = PyodideLockSpec(**lock_data) +def test_check_wheel_filenames(example_lock_data): + spec = PyodideLockSpec(**example_lock_data) spec.check_wheel_filenames() - lock_data["packages"]["numpy"]["name"] = "numpy2" # type: ignore[index] - spec = PyodideLockSpec(**lock_data) + example_lock_data["packages"]["numpy"]["name"] = "numpy2" # type: ignore[index] + spec = PyodideLockSpec(**example_lock_data) msg = ( ".*check_wheel_filenames failed.*\n.*numpy:\n.*" "Package name in wheel filename 'numpy' does not match 'numpy2'" @@ -70,8 +46,8 @@ def test_check_wheel_filenames(): with pytest.raises(ValueError, match=msg): spec.check_wheel_filenames() - lock_data["packages"]["numpy"]["version"] = "0.2.3" # type: ignore[index] - spec = PyodideLockSpec(**lock_data) + example_lock_data["packages"]["numpy"]["version"] = "0.2.3" # type: ignore[index] + spec = PyodideLockSpec(**example_lock_data) msg = ( ".*check_wheel_filenames failed.*\n.*numpy:\n.*" "Package name in wheel filename 'numpy' does not match 'numpy2'\n.*" @@ -82,11 +58,10 @@ def test_check_wheel_filenames(): spec.check_wheel_filenames() -def test_to_json_indent(tmp_path): - lock_data = deepcopy(LOCK_EXAMPLE) +def test_to_json_indent(tmp_path, example_lock_data): target_path = tmp_path / "pyodide-lock.json" - spec = PyodideLockSpec(**lock_data) + spec = PyodideLockSpec(**example_lock_data) spec.to_json(target_path) assert "\n" not in target_path.read_text() @@ -98,30 +73,30 @@ def test_to_json_indent(tmp_path): assert "\n" in target_path.read_text() -def test_update_sha256(monkeypatch): +def test_update_sha256(monkeypatch, example_lock_data): monkeypatch.setattr("pyodide_lock.utils._generate_package_hash", lambda x: "abcd") - lock_data = deepcopy(LOCK_EXAMPLE) - lock_data["packages"]["numpy"]["sha256"] = "0" # type: ignore[index] - spec = PyodideLockSpec(**lock_data) + example_lock_data["packages"]["numpy"]["sha256"] = "0" # type: ignore[index] + spec = PyodideLockSpec(**example_lock_data) assert spec.packages["numpy"].sha256 == "0" update_package_sha256(spec.packages["numpy"], Path("/some/path")) assert spec.packages["numpy"].sha256 == "abcd" -def test_extra_config_forbidden(): +def test_extra_config_forbidden(example_lock_data): from pydantic import ValidationError - lock_data = deepcopy(LOCK_EXAMPLE) - info_data = deepcopy(lock_data["info"]) - package_data = deepcopy(lock_data["packages"]["numpy"]) # type: ignore[index] + info_data = deepcopy(example_lock_data["info"]) + package_data = deepcopy( + example_lock_data["packages"]["numpy"] + ) # type: ignore[index] - lock_data["extra"] = "extra" + example_lock_data["extra"] = "extra" info_data["extra"] = "extra" # type: ignore[index] package_data["extra"] = "extra" with pytest.raises(ValidationError, match="extra fields not permitted"): - PyodideLockSpec(**lock_data) + PyodideLockSpec(**example_lock_data) with pytest.raises(ValidationError, match="extra fields not permitted"): InfoSpec(**info_data) # type: ignore[arg-type] diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 08464c6..0aa0558 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -1,18 +1,11 @@ -import json import zipfile -from copy import deepcopy -from dataclasses import asdict, dataclass from pathlib import Path -from tempfile import TemporaryDirectory -import build import pytest -from test_spec import LOCK_EXAMPLE -from pyodide_lock import PackageSpec, PyodideLockSpec +from pyodide_lock import PackageSpec from pyodide_lock.utils import ( _generate_package_hash, - _get_marker_environment, add_wheels_to_spec, ) @@ -22,209 +15,92 @@ DIST = HERE.parent / "dist" WHEEL = next(DIST.glob("*.whl")) if DIST.exists() else None -# marker environment for testing -_ENV = _get_marker_environment(**LOCK_EXAMPLE["info"]) # type:ignore[arg-type] -# marker environment for testing, filtered only to numerical values -_ENV_NUM = {k: v for k, v in _ENV.items() if v[0] in "0123456789"} -MARKER_EXAMPLES_NOT_NEEDED = ( - [ - 'requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < "2.7"', - 'argparse;python_version<"2.7"', - ] - + [f'Not.expected ; {k} != "{v}"' for k, v in _ENV.items()] - + [f'Not.expected ; {k} > "{v}"' for k, v in _ENV_NUM.items()] -) - - -MARKER_EXAMPLES_NEEDED = ( - [ - 'a;python_version>="3.5"', - 'b;sys_platform=="emscripten"', - ] - + [f'c_{k}; {k} == "{v}"' for k, v in _ENV.items()] - + [f'd_{k} ; {k} <= "{v}"' for k, v in _ENV_NUM.items()] -) - - -# build a wheel -def make_test_wheel( - dir: Path, - package_name: str, - deps: list[str] | None = None, - optional_deps: dict[str, list[str]] | None = None, - modules: list[str] | None = None, -): - package_dir = dir / package_name - package_dir.mkdir() - if modules is None: - modules = [package_name] - for m in modules: - (package_dir / f"{m}.py").write_text("") - toml = package_dir / "pyproject.toml" - if deps is None: - deps = [] - - all_deps = json.dumps(deps) - if optional_deps: - all_optional_deps = "[project.optional-dependencies]\n" + "\n".join( - [x + "=" + json.dumps(optional_deps[x]) for x in optional_deps.keys()] - ) - else: - all_optional_deps = "" - toml_text = f""" -[build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" - -[project] -name = "{package_name}" -description = "{package_name} example package" -version = "1.0.0" -authors = [ - {{ name = "Bob Jones", email = "bobjones@nowhere.nowhere" }} -] -dependencies = { - all_deps -} - -{ all_optional_deps } - -""" - toml.write_text(toml_text) - builder = build.ProjectBuilder(package_dir) - return Path(builder.build("wheel", dir / "dist")) - - -@pytest.fixture(scope="module") -def test_wheel_list(): - @dataclass - class TestWheel: - package_name: str - modules: list[str] | None = None - deps: list[str] | None = None - optional_deps: dict[str, list[str]] | None = None - - # a set of test wheels - note that names are non-canonicalized - # deliberately to test this - test_wheels: list[TestWheel] = [ - TestWheel(package_name="py-one", modules=["one"]), - TestWheel(package_name="NEeds-one", deps=["py_one"]), - TestWheel(package_name="nEEds-one-opt", optional_deps={"with_one": ["py_One"]}), - TestWheel( - package_name="test-extra_dependencies", deps=["needs-one-opt[with_one]"] - ), - TestWheel(package_name="failure", deps=["two"]), - TestWheel( - package_name="markers_not_needed_test", deps=MARKER_EXAMPLES_NOT_NEEDED - ), - TestWheel(package_name="markers_needed_test", deps=MARKER_EXAMPLES_NEEDED), - ] - - with TemporaryDirectory() as tmpdir: - path_temp = Path(tmpdir) - path_temp.mkdir(exist_ok=True) - all_wheels = [] - for wheel_data in test_wheels: - all_wheels.append(make_test_wheel(path_temp, **asdict(wheel_data))) - yield all_wheels - - -def test_add_one(test_wheel_list): - lock_data = deepcopy(LOCK_EXAMPLE) - spec = PyodideLockSpec(**lock_data) - - add_wheels_to_spec(spec, test_wheel_list[0:1]) +def test_add_one(test_wheel_list, example_lock_spec): + add_wheels_to_spec(example_lock_spec, test_wheel_list[0:1]) # py_one only should get added - assert spec.packages["py-one"].imports == ["one"] + assert example_lock_spec.packages["py-one"].imports == ["one"] -def test_add_simple_deps(test_wheel_list): - lock_data = deepcopy(LOCK_EXAMPLE) - spec = PyodideLockSpec(**lock_data) - add_wheels_to_spec(spec, test_wheel_list[0:3]) +def test_add_simple_deps(test_wheel_list, example_lock_spec): + add_wheels_to_spec(example_lock_spec, test_wheel_list[0:3]) # py_one, needs_one and needs_one_opt should get added - assert "py-one" in spec.packages - assert "needs-one" in spec.packages - assert "needs-one-opt" in spec.packages + assert "py-one" in example_lock_spec.packages + assert "needs-one" in example_lock_spec.packages + assert "needs-one-opt" in example_lock_spec.packages # needs one opt should not depend on py_one - assert spec.packages["needs-one-opt"].depends == [] + assert example_lock_spec.packages["needs-one-opt"].depends == [] # needs one should depend on py_one - assert spec.packages["needs-one"].depends == ["py-one"] + assert example_lock_spec.packages["needs-one"].depends == ["py-one"] -def test_add_deps_with_extras(test_wheel_list): - lock_data = deepcopy(LOCK_EXAMPLE) - spec = PyodideLockSpec(**lock_data) - add_wheels_to_spec(spec, test_wheel_list[0:4]) +def test_add_deps_with_extras(test_wheel_list, example_lock_spec): + add_wheels_to_spec(example_lock_spec, test_wheel_list[0:4]) # py_one, needs_one, needs_one_opt and test_extra_dependencies should get added # because of the extra dependency in test_extra_dependencies, # needs_one_opt should now depend on one - assert "test-extra-dependencies" in spec.packages - assert spec.packages["needs-one-opt"].depends == ["py-one"] + assert "test-extra-dependencies" in example_lock_spec.packages + assert example_lock_spec.packages["needs-one-opt"].depends == ["py-one"] -def test_missing_dep(test_wheel_list): - lock_data = deepcopy(LOCK_EXAMPLE) - spec = PyodideLockSpec(**lock_data) +def test_missing_dep(test_wheel_list, example_lock_spec): # this has a package with a missing dependency so should fail with pytest.raises(RuntimeError): - add_wheels_to_spec(spec, test_wheel_list[0:5]) + add_wheels_to_spec(example_lock_spec, test_wheel_list[0:5]) -def test_path_rewriting(test_wheel_list): - lock_data = deepcopy(LOCK_EXAMPLE) - spec = PyodideLockSpec(**lock_data) - add_wheels_to_spec(spec, test_wheel_list[0:3], base_url="http://www.nowhere.org/") +def test_url_rewriting(test_wheel_list, example_lock_spec): + add_wheels_to_spec( + example_lock_spec, test_wheel_list[0:3], base_url="http://www.nowhere.org/" + ) # py_one, needs_one and needs_one_opt should get added - assert "py-one" in spec.packages - assert "needs-one" in spec.packages - assert "needs-one-opt" in spec.packages - assert spec.packages["py-one"].file_name.startswith("http://www.nowhere.org/py_one") + assert "py-one" in example_lock_spec.packages + assert "needs-one" in example_lock_spec.packages + assert "needs-one-opt" in example_lock_spec.packages + assert example_lock_spec.packages["py-one"].file_name.startswith( + "http://www.nowhere.org/py_one" + ) - lock_data = deepcopy(LOCK_EXAMPLE) - spec = PyodideLockSpec(**lock_data) - # this should add the base path "dist" to the file name + +def test_base_relative_path(test_wheel_list, example_lock_spec): + # this should make all the file names relative to the + # parent path of the wheels (which is "dist") add_wheels_to_spec( - spec, + example_lock_spec, test_wheel_list[0:3], base_url="http://www.nowhere.org/", base_path=test_wheel_list[0].parent.parent, ) # py_one, needs_one and needs_one_opt should get added - assert "py-one" in spec.packages - assert "needs-one" in spec.packages - assert "needs-one-opt" in spec.packages - assert spec.packages["needs-one-opt"].file_name.startswith( + assert "py-one" in example_lock_spec.packages + assert "needs-one" in example_lock_spec.packages + assert "needs-one-opt" in example_lock_spec.packages + assert example_lock_spec.packages["needs-one-opt"].file_name.startswith( "http://www.nowhere.org/dist/nEEds" ) # all requirements markers should not be needed, so dependencies should be empty -def test_markers_not_needed(test_wheel_list): - lock_data = deepcopy(LOCK_EXAMPLE) - spec = PyodideLockSpec(**lock_data) - add_wheels_to_spec(spec, test_wheel_list[5:6]) - assert spec.packages["markers-not-needed-test"].depends == [] +def test_markers_not_needed(test_wheel_list, example_lock_spec): + add_wheels_to_spec(example_lock_spec, test_wheel_list[5:6]) + assert example_lock_spec.packages["markers-not-needed-test"].depends == [] # all requirements markers should be needed, -# so returned dependencies should be the same length as MARKER_EXAMPLES_NEEDED -def test_markers_needed(test_wheel_list): - lock_data = deepcopy(LOCK_EXAMPLE) - spec = PyodideLockSpec(**lock_data) - add_wheels_to_spec(spec, test_wheel_list[6:7], ignore_missing_dependencies=True) - assert len(spec.packages["markers-needed-test"].depends) == len( - MARKER_EXAMPLES_NEEDED +# so returned dependencies should be the same length as marker_examples_needed +def test_markers_needed(test_wheel_list, example_lock_spec, marker_examples_needed): + add_wheels_to_spec( + example_lock_spec, test_wheel_list[6:7], ignore_missing_dependencies=True + ) + assert len(example_lock_spec.packages["markers-needed-test"].depends) == len( + marker_examples_needed ) @pytest.mark.skipif(WHEEL is None, reason="wheel test requires a built wheel") -def test_self_wheel(): +def test_self_wheel(example_lock_spec): assert WHEEL is not None - lock_data = deepcopy(LOCK_EXAMPLE) - spec = PyodideLockSpec(**lock_data) - add_wheels_to_spec(spec, [WHEEL], ignore_missing_dependencies=True) + add_wheels_to_spec(example_lock_spec, [WHEEL], ignore_missing_dependencies=True) expected = PackageSpec( name="pyodide-lock", @@ -239,18 +115,16 @@ def test_self_wheel(): shared_library=False, ) - assert spec.packages["pyodide-lock"] == expected + assert example_lock_spec.packages["pyodide-lock"] == expected -def test_not_wheel(tmp_path): - lock_data = deepcopy(LOCK_EXAMPLE) - spec = PyodideLockSpec(**lock_data) +def test_not_wheel(tmp_path, example_lock_spec): wheel = tmp_path / "not_a_wheel-1.0.0-py3-none-any.whl" with zipfile.ZipFile(wheel, "w") as whlzip: whlzip.writestr("README.md", data="Not a wheel") with pytest.raises(RuntimeError, match="metadata"): - add_wheels_to_spec(spec, [wheel]) + add_wheels_to_spec(example_lock_spec, [wheel]) @pytest.mark.parametrize( @@ -260,11 +134,9 @@ def test_not_wheel(tmp_path): "bad_version_for_a_wheel-a.0.0-py3-none-any.whl", ], ) -def test_bad_names(tmp_path, bad_name): - lock_data = deepcopy(LOCK_EXAMPLE) - spec = PyodideLockSpec(**lock_data) +def test_bad_names(tmp_path, bad_name, example_lock_spec): wheel = tmp_path / bad_name with zipfile.ZipFile(wheel, "w") as whlzip: whlzip.writestr("README.md", data="Not a wheel") with pytest.raises(RuntimeError, match="Wheel filename"): - add_wheels_to_spec(spec, [wheel]) + add_wheels_to_spec(example_lock_spec, [wheel]) From ba8ac4c94cc6529c013dd3fa15aacfc099b304a8 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 13 Oct 2023 12:19:17 +0100 Subject: [PATCH 20/22] review fixes --- pyodide_lock/cli.py | 102 ++++++++++++++++++++---------------------- pyodide_lock/spec.py | 5 +-- pyodide_lock/utils.py | 82 +++++++++++++++++---------------- tests/conftest.py | 5 ++- tests/test_wheel.py | 41 +++++++++++++++++ 5 files changed, 138 insertions(+), 97 deletions(-) diff --git a/pyodide_lock/cli.py b/pyodide_lock/cli.py index 9fb3fc6..af3c28e 100644 --- a/pyodide_lock/cli.py +++ b/pyodide_lock/cli.py @@ -1,61 +1,57 @@ from pathlib import Path -try: - import typer +import typer - from .spec import PyodideLockSpec - from .utils import add_wheels_to_spec +from .spec import PyodideLockSpec +from .utils import add_wheels_to_spec - main = typer.Typer() +main = typer.Typer(help="manipulate pyodide-lock.json lockfiles.") - @main.command() - def add_wheels( - wheels: list[Path], - ignore_missing_dependencies: bool = typer.Option( - help="If this is true, dependencies " - "which are not in the original lockfile or " - "the added wheels will be added to the lockfile. " - "Warning: This will allow a broken lockfile to " - "be created.", - default=False, - ), - in_lockfile: Path = typer.Option( - help="Source lockfile (input)", default=Path("pyodide-lock.json") - ), - out_lockfile: Path = typer.Option( - help="Updated lockfile (output)", default=Path("pyodide-lock-new.json") - ), - base_path: Path = typer.Option( - help="Base path for wheels - wheel file " - "names will be created relative to this path.", - default=None, - ), - wheel_url: str = typer.Option( - help="Base url which will be appended to the wheel location." - "Use this if you are hosting these wheels on a different " - "server to core pyodide packages", - default="", - ), - ): - """Add a set of package wheels to an existing pyodide-lock.json and - write it out to pyodide-lock-new.json - Each package in the wheel will be added to the output lockfile, - including resolution of dependencies in the lock file. By default - this will fail if a dependency isn't available in either the - existing lock file, or in the set of new wheels. +@main.command() +def add_wheels( + wheels: list[Path], + ignore_missing_dependencies: bool = typer.Option( + help="If this is true, dependencies " + "which are not in the original lockfile or " + "the added wheels will be added to the lockfile. " + "Warning: This will allow a broken lockfile to " + "be created.", + default=False, + ), + input: Path = typer.Option( + help="Source lockfile", default=Path("pyodide-lock.json") + ), + output: Path = typer.Option( + help="Updated lockfile", default=Path("pyodide-lock-new.json") + ), + base_path: Path = typer.Option( + help="Base path for wheels - wheel file " + "names will be created relative to this path.", + default=None, + ), + wheel_url: str = typer.Option( + help="Base url which will be appended to the wheel location." + "Use this if you are hosting these wheels on a different " + "server to core pyodide packages", + default="", + ), +): + """Add a set of package wheels to an existing pyodide-lock.json and + write it out to pyodide-lock-new.json - """ - sp = PyodideLockSpec.from_json(in_lockfile) - add_wheels_to_spec( - sp, - wheels, - base_path=base_path, - base_url=wheel_url, - ignore_missing_dependencies=ignore_missing_dependencies, - ) - sp.to_json(out_lockfile) + Each package in the wheel will be added to the output lockfile, + including resolution of dependencies in the lock file. By default + this will fail if a dependency isn't available in either the + existing lock file, or in the set of new wheels. -except ImportError: - pass - # no typer = no cli + """ + sp = PyodideLockSpec.from_json(input) + add_wheels_to_spec( + sp, + wheels, + base_path=base_path, + base_url=wheel_url, + ignore_missing_dependencies=ignore_missing_dependencies, + ) + sp.to_json(output) diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index b287174..b91a4ec 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -1,12 +1,9 @@ import json from pathlib import Path -from typing import TYPE_CHECKING, Literal +from typing import Literal from pydantic import BaseModel, Extra, Field -if TYPE_CHECKING: - pass - class InfoSpec(BaseModel): arch: Literal["wasm32", "wasm64"] = "wasm32" diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index d4a7db0..ba03e2c 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -44,18 +44,12 @@ def parse_top_level_import_name(whlfile: Path) -> list[str] | None: whlzip = zipfile.Path(whlfile) - # if there is a directory with .dist_info at the end with a top_level.txt file - # then just use that - for subdir in whlzip.iterdir(): - if subdir.name.endswith(".dist-info"): - top_level_path = subdir / "top_level.txt" - if top_level_path.exists(): - return top_level_path.read_text().splitlines() - - # If there is no top_level.txt file, we will find top level imports by + # We will find top level imports by # 1) a python file on a top-level directory # 2) a sub directory with __init__.py # following: https://github.com/pypa/setuptools/blob/d680efc8b4cd9aa388d07d3e298b870d26e9e04b/setuptools/discovery.py#L122 + # - n.b. this is more reliable than using top-level.txt which is + # sometimes broken top_level_imports = [] for subdir in whlzip.iterdir(): if subdir.is_file() and subdir.name.endswith(".py"): @@ -63,7 +57,6 @@ def parse_top_level_import_name(whlfile: Path) -> list[str] | None: elif subdir.is_dir() and _valid_package_name(subdir.name): if _has_python_file(subdir): top_level_imports.append(subdir.name) - if not top_level_imports: logger.warning( f"WARNING: failed to parse top level import name from {whlfile}." @@ -166,20 +159,25 @@ def add_wheels_to_spec( ) -> None: """Add a list of wheel files to this pyodide-lock.json - Args: - wheel_files (list[Path]): A list of wheel files to import. - base_path (Path | None, optional): - Filenames are stored relative to this base path. By default the - filename is stored relative to the path of the first wheel file - in the list. - - base_url (str, optional): - The base URL stored in the pyodide-lock.json. By default this - is empty which means that wheels must be stored in the same folder - as the core pyodide packages you are using. If you want to store - your custom wheels somewhere else, set this base_url to point to it. + Parameters: + wheel_files : list[Path] + A list of wheel files to import. + base_path : Path | None, optional + Filenames are stored relative to this base path. By default the + filename is stored relative to the path of the first wheel file + in the list. + base_url : str, optional + The base URL stored in the pyodide-lock.json. By default this + is empty which means that wheels must be stored in the same folder + as the core pyodide packages you are using. If you want to store + your custom wheels somewhere else, set this base_url to point to it. + ignore_missing_dependencies: bool, optional + If this is set to True, any dependencies not found in the lock file + or the set of wheels being added will be added to the spec. This is + not 100% reliable, because it ignores any extras and does not do any + sub-dependency or version resolution. """ - if len(wheel_files) <= 0: + if not wheel_files: return wheel_files = [f.resolve() for f in wheel_files] if base_path is None: @@ -254,7 +252,7 @@ def _fix_extra_dep( extra_req: "Requirement", new_packages: dict[str, PackageSpec], ignore_missing_dependencies: bool, -): +) -> list["Requirement"]: from packaging.utils import canonicalize_name requirements_with_extras = [] @@ -303,26 +301,14 @@ def _set_package_paths( p.file_name = base_url + str(relative_path) -def package_spec_from_wheel(path: Path, info: InfoSpec) -> "PackageSpec": - """Build a package spec from an on-disk wheel. - - Warning - to reliably handle dependencies, we need: - 1) To have access to all the wheels being added at once (to handle extras) - 2) To know whether dependencies are available in the combined lockfile. - 3) To fix up wheel urls and paths consistently - - This is called by add_wheels_to_spec - """ +def _check_wheel_compatible(path: Path, info: InfoSpec) -> None: from packaging.utils import ( InvalidWheelFilename, - canonicalize_name, parse_wheel_filename, ) from packaging.version import InvalidVersion from packaging.version import parse as version_parse - path = path.absolute() - # throw an error if this is an incompatible wheel target_python = version_parse(info.python) target_platform = info.platform + "_" + info.arch try: @@ -349,10 +335,30 @@ def package_spec_from_wheel(path: Path, info: InfoSpec) -> "PackageSpec": tag_match = True if not tag_match: raise RuntimeError( - f"Package tags {tags} don't match Python version in lockfile:" + f"Package tags for {path} don't match Python version in lockfile:" f"Lockfile python {target_python.major}.{target_python.minor}" f"on platform {target_platform} ({python_binary_abi})" ) + + +def package_spec_from_wheel(path: Path, info: InfoSpec) -> "PackageSpec": + """Build a package spec from an on-disk wheel. + + Warning - to reliably handle dependencies, we need: + 1) To have access to all the wheels being added at once (to handle extras) + 2) To know whether dependencies are available in the combined lockfile. + 3) To fix up wheel urls and paths consistently + + This is called by add_wheels_to_spec + """ + from packaging.utils import ( + canonicalize_name, + ) + + path = path.absolute() + # throw an error if this is an incompatible wheel + + _check_wheel_compatible(path, info) metadata = _wheel_metadata(path) if not metadata: diff --git a/tests/conftest.py b/tests/conftest.py index 6c04f7e..602fe0a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ import build import pytest +from packaging.utils import canonicalize_name from pyodide_lock import PyodideLockSpec from pyodide_lock.utils import _get_marker_environment @@ -87,8 +88,8 @@ def make_test_wheel( ): package_dir = dir / package_name package_dir.mkdir() - if modules is None: - modules = [package_name] + if not modules: + modules = [canonicalize_name(package_name).replace("-", "_")] for m in modules: (package_dir / f"{m}.py").write_text("") toml = package_dir / "pyproject.toml" diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 0aa0558..adc718b 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -2,9 +2,11 @@ from pathlib import Path import pytest +from packaging.version import parse as version_parse from pyodide_lock import PackageSpec from pyodide_lock.utils import ( + _check_wheel_compatible, _generate_package_hash, add_wheels_to_spec, ) @@ -140,3 +142,42 @@ def test_bad_names(tmp_path, bad_name, example_lock_spec): whlzip.writestr("README.md", data="Not a wheel") with pytest.raises(RuntimeError, match="Wheel filename"): add_wheels_to_spec(example_lock_spec, [wheel]) + + +def test_wheel_compatibility_checking(example_lock_spec): + target_python = version_parse(example_lock_spec.info.python) + python_tag = f"py{target_python.major}{target_python.minor}" + cpython_tag = f"cp{target_python.major}{target_python.minor}" + emscripten_tag = example_lock_spec.info.platform + "_" + example_lock_spec.info.arch + + # pure python 3 wheel + _check_wheel_compatible( + Path("test_wheel-1.0.0-py3-none-any.whl"), example_lock_spec.info + ) + # pure python 3.X wheel + _check_wheel_compatible( + Path(f"test_wheel-1.0.0-{python_tag}-none-any.whl"), example_lock_spec.info + ) + # pure python 2 or 3 wheel + _check_wheel_compatible( + Path("test_wheel-1.0.0-py2.py3-none-any.whl"), example_lock_spec.info + ) + # cpython emscripten correct version + _check_wheel_compatible( + Path(f"test_wheel-1.0.0-{cpython_tag}-{cpython_tag}-{emscripten_tag}.whl"), + example_lock_spec.info, + ) + with pytest.raises(RuntimeError): + # cpython emscripten incorrect version + _check_wheel_compatible( + Path( + f"test_wheel-1.0.0-{cpython_tag}-{cpython_tag}-emscripten_3_1_2_wasm32.whl" + ), + example_lock_spec.info, + ) + with pytest.raises(RuntimeError): + # a linux wheel + _check_wheel_compatible( + Path(f"test_wheel-1.0.0-{cpython_tag}-{cpython_tag}-linux_x86_64.whl"), + example_lock_spec.info, + ) From 20f0f79760d1da0728beb4df483c38585f5c61fe Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 13 Oct 2023 12:26:24 +0100 Subject: [PATCH 21/22] review updates --- pyodide_lock/utils.py | 11 +++++++---- tests/test_wheel.py | 29 ++++++++++++++++------------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index ba03e2c..fbfeb8c 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -156,8 +156,9 @@ def add_wheels_to_spec( base_path: Path | None = None, base_url: str = "", ignore_missing_dependencies: bool = False, -) -> None: - """Add a list of wheel files to this pyodide-lock.json +) -> PyodideLockSpec: + """Add a list of wheel files to this pyodide-lock.json and return a + new PyodideLockSpec Parameters: wheel_files : list[Path] @@ -177,8 +178,9 @@ def add_wheels_to_spec( not 100% reliable, because it ignores any extras and does not do any sub-dependency or version resolution. """ + new_spec = lock_spec.copy(deep=True) if not wheel_files: - return + return new_spec wheel_files = [f.resolve() for f in wheel_files] if base_path is None: base_path = wheel_files[0].parent @@ -193,7 +195,8 @@ def add_wheels_to_spec( _fix_new_package_deps(lock_spec, new_packages, ignore_missing_dependencies) _set_package_paths(new_packages, base_path, base_url) - lock_spec.packages |= new_packages + new_spec.packages |= new_packages + return new_spec def _fix_new_package_deps( diff --git a/tests/test_wheel.py b/tests/test_wheel.py index adc718b..265da85 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -19,13 +19,14 @@ def test_add_one(test_wheel_list, example_lock_spec): - add_wheels_to_spec(example_lock_spec, test_wheel_list[0:1]) - # py_one only should get added - assert example_lock_spec.packages["py-one"].imports == ["one"] + new_lock_spec = add_wheels_to_spec(example_lock_spec, test_wheel_list[0:1]) + # py_one only should get added to the new spec + assert new_lock_spec.packages["py-one"].imports == ["one"] + assert "py-one" not in example_lock_spec.packages def test_add_simple_deps(test_wheel_list, example_lock_spec): - add_wheels_to_spec(example_lock_spec, test_wheel_list[0:3]) + example_lock_spec = add_wheels_to_spec(example_lock_spec, test_wheel_list[0:3]) # py_one, needs_one and needs_one_opt should get added assert "py-one" in example_lock_spec.packages assert "needs-one" in example_lock_spec.packages @@ -37,7 +38,7 @@ def test_add_simple_deps(test_wheel_list, example_lock_spec): def test_add_deps_with_extras(test_wheel_list, example_lock_spec): - add_wheels_to_spec(example_lock_spec, test_wheel_list[0:4]) + example_lock_spec = add_wheels_to_spec(example_lock_spec, test_wheel_list[0:4]) # py_one, needs_one, needs_one_opt and test_extra_dependencies should get added # because of the extra dependency in test_extra_dependencies, # needs_one_opt should now depend on one @@ -48,11 +49,11 @@ def test_add_deps_with_extras(test_wheel_list, example_lock_spec): def test_missing_dep(test_wheel_list, example_lock_spec): # this has a package with a missing dependency so should fail with pytest.raises(RuntimeError): - add_wheels_to_spec(example_lock_spec, test_wheel_list[0:5]) + example_lock_spec = add_wheels_to_spec(example_lock_spec, test_wheel_list[0:5]) def test_url_rewriting(test_wheel_list, example_lock_spec): - add_wheels_to_spec( + example_lock_spec = add_wheels_to_spec( example_lock_spec, test_wheel_list[0:3], base_url="http://www.nowhere.org/" ) # py_one, needs_one and needs_one_opt should get added @@ -67,7 +68,7 @@ def test_url_rewriting(test_wheel_list, example_lock_spec): def test_base_relative_path(test_wheel_list, example_lock_spec): # this should make all the file names relative to the # parent path of the wheels (which is "dist") - add_wheels_to_spec( + example_lock_spec = add_wheels_to_spec( example_lock_spec, test_wheel_list[0:3], base_url="http://www.nowhere.org/", @@ -84,14 +85,14 @@ def test_base_relative_path(test_wheel_list, example_lock_spec): # all requirements markers should not be needed, so dependencies should be empty def test_markers_not_needed(test_wheel_list, example_lock_spec): - add_wheels_to_spec(example_lock_spec, test_wheel_list[5:6]) + example_lock_spec = add_wheels_to_spec(example_lock_spec, test_wheel_list[5:6]) assert example_lock_spec.packages["markers-not-needed-test"].depends == [] # all requirements markers should be needed, # so returned dependencies should be the same length as marker_examples_needed def test_markers_needed(test_wheel_list, example_lock_spec, marker_examples_needed): - add_wheels_to_spec( + example_lock_spec = add_wheels_to_spec( example_lock_spec, test_wheel_list[6:7], ignore_missing_dependencies=True ) assert len(example_lock_spec.packages["markers-needed-test"].depends) == len( @@ -102,7 +103,9 @@ def test_markers_needed(test_wheel_list, example_lock_spec, marker_examples_need @pytest.mark.skipif(WHEEL is None, reason="wheel test requires a built wheel") def test_self_wheel(example_lock_spec): assert WHEEL is not None - add_wheels_to_spec(example_lock_spec, [WHEEL], ignore_missing_dependencies=True) + example_lock_spec = add_wheels_to_spec( + example_lock_spec, [WHEEL], ignore_missing_dependencies=True + ) expected = PackageSpec( name="pyodide-lock", @@ -126,7 +129,7 @@ def test_not_wheel(tmp_path, example_lock_spec): whlzip.writestr("README.md", data="Not a wheel") with pytest.raises(RuntimeError, match="metadata"): - add_wheels_to_spec(example_lock_spec, [wheel]) + example_lock_spec = add_wheels_to_spec(example_lock_spec, [wheel]) @pytest.mark.parametrize( @@ -141,7 +144,7 @@ def test_bad_names(tmp_path, bad_name, example_lock_spec): with zipfile.ZipFile(wheel, "w") as whlzip: whlzip.writestr("README.md", data="Not a wheel") with pytest.raises(RuntimeError, match="Wheel filename"): - add_wheels_to_spec(example_lock_spec, [wheel]) + example_lock_spec = add_wheels_to_spec(example_lock_spec, [wheel]) def test_wheel_compatibility_checking(example_lock_spec): From 5236af026b1fd2f234bcd5b8e663e7d3942dfdae Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 13 Oct 2023 12:32:06 +0100 Subject: [PATCH 22/22] type info --- pyodide_lock/utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index fbfeb8c..a201850 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -1,6 +1,7 @@ import hashlib import logging import re +import sys import zipfile from collections import deque from functools import cache @@ -101,18 +102,18 @@ def _generate_package_hash(full_path: Path) -> str: return sha256_hash.hexdigest() -def _get_marker_environment(platform: str, version: str, arch: str, python: str): +def _get_marker_environment( + platform: str, version: str, arch: str, python: str +) -> dict[str, str]: """ Get the marker environment for this pyodide-lock file. If running inside pyodide it returns the current marker environment. """ - - try: - exec("import pyodide") + if "pyodide" in sys.modules: from packaging.markers import default_environment return default_environment() - except ImportError: + else: marker_env = _PYODIDE_MARKER_ENV.copy() from packaging.version import parse as version_parse @@ -129,7 +130,7 @@ def _get_marker_environment(platform: str, version: str, arch: str, python: str) @cache -def _wheel_metadata(path: Path): +def _wheel_metadata(path: Path) -> "Distribution": """Cached wheel metadata to save opening the file multiple times""" from pkginfo import get_metadata @@ -344,7 +345,7 @@ def _check_wheel_compatible(path: Path, info: InfoSpec) -> None: ) -def package_spec_from_wheel(path: Path, info: InfoSpec) -> "PackageSpec": +def package_spec_from_wheel(path: Path, info: InfoSpec) -> PackageSpec: """Build a package spec from an on-disk wheel. Warning - to reliably handle dependencies, we need: