diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index c2f0c58..8aa83e6 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -2,9 +2,10 @@ from pathlib import Path from typing import Literal -from pydantic import BaseModel, Extra +from pydantic import BaseModel, Extra, Field from .utils import ( + _add_required, _generate_package_hash, _wheel_depends, parse_top_level_import_name, @@ -12,32 +13,113 @@ class InfoSpec(BaseModel): - arch: Literal["wasm32", "wasm64"] = "wasm32" - platform: str - version: str - python: str + arch: Literal["wasm32", "wasm64"] = Field( + default="wasm32", + description=( + "the short name for the compiled architecture, available in " + "dependency markers as `platform_machine`" + ), + ) + platform: str = Field( + description=( + "the emscripten virtual machine for which this distribution is " + " compiled, not available directly in a dependency marker: use e.g. " + """`plaform_system == "Emscripten" and platform_release == "3.1.45"`""" + ), + examples=["emscripten_3_1_32", "emscripten_3_1_45"], + ) + version: str = Field( + description="the PEP 440 version of pyodide", + examples=["0.24.1", "0.23.3"], + ) + python: str = Field( + description=( + "the version of python for which this lockfile is valid, available in " + "version markers as `platform_machine`" + ), + examples=["3.11.2", "3.11.3"], + ) class Config: extra = Extra.forbid + schema_extra = _add_required( + "arch", + description=( + "the execution environment in which the packages in this lockfile " + "can be installed" + ), + ) + class PackageSpec(BaseModel): - name: str - version: str - file_name: str - install_dir: str - sha256: str = "" + name: str = Field( + description="the verbatim name as found in the package's metadata", + examples=["pyodide-lock", "PyYAML", "ruamel.yaml"], + ) + version: str = Field( + description="the reported version of the package", + examples=["0.1.0", "1.0.0a0", "1.0.0a0.post1"], + ) + file_name: str = Field( + format="uri-reference", + description="the URL of the file", + examples=[ + "pyodide_lock-0.1.0-py3-none-any.whl", + "https://files.pythonhosted.org/packages/py3/m/micropip/micropip-0.5.0-py3-none-any.whl", + ], + ) + install_dir: str = Field( + default="site", + description="the file system destination for a package's data", + examples=["dynlib", "stdlib"], + ) + sha256: str = Field(description="the SHA256 cryptographic hash of the file") package_type: Literal[ "package", "cpython_module", "shared_library", "static_library" - ] = "package" - imports: list[str] = [] - depends: list[str] = [] - unvendored_tests: bool = False + ] = Field( + default="package", + description="the top-level kind of content provided by this package", + ) + imports: list[str] = Field( + default=[], + description=( + "the importable names provided by this package." + "note that PEP 420 namespace packages will likely not be correctly found." + ), + ) + depends: list[str] = Field( + default=[], + unique_items=True, + description=( + "package names that must be installed when this package in installed" + ), + ) + unvendored_tests: bool = Field( + default=False, + description=( + "whether the package's tests folder have been repackaged " + "as a separate archive" + ), + ) # This field is deprecated - shared_library: bool = False + shared_library: bool = Field( + default=False, + deprecated=True, + description=( + "(deprecated) whether this package is a shared library. " + "replaced with `package_type: shared_library`" + ), + ) class Config: extra = Extra.forbid + schema_extra = _add_required( + "depends", + "imports", + "install_dir", + description="a single pyodide-compatible file", + ) @classmethod def from_wheel( @@ -78,11 +160,27 @@ def update_sha256(self, path: Path) -> "PackageSpec": class PyodideLockSpec(BaseModel): """A specification for the pyodide-lock.json file.""" - info: InfoSpec - packages: dict[str, PackageSpec] + info: InfoSpec = Field( + description=( + "the execution environment in which the packages in this lockfile " + "can be installable" + ) + ) + packages: dict[str, PackageSpec] = Field( + default={}, + description="a set of packages keyed by name", + ) class Config: extra = Extra.forbid + schema_extra = { + "$schema": "https://json-schema.org/draft/2019-09/schema#", + "$id": ("https://pyodide.org/schema/pyodide-lock/v0-lockfile.schema.json"), + "description": ( + "a description of a viable pyodide runtime environment, " + "as defined by pyodide-lock" + ), + } @classmethod def from_json(cls, path: Path) -> "PyodideLockSpec": diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index dfbefd3..16ae9a2 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -4,8 +4,9 @@ import sys import zipfile from collections import deque +from collections.abc import Callable from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from pkginfo import Distribution @@ -130,3 +131,13 @@ def _wheel_depends( depends += [canonicalize_name(req.name)] return sorted(set(depends)) + + +def _add_required( + *field_names: str, **extra: Any +) -> Callable[[dict[str, Any], Any], None]: + def add_required(schema: dict[str, Any], *args: Any) -> None: + schema["required"] = sorted([*field_names, *schema.get("required", [])]) + schema.update(extra) + + return add_required diff --git a/pyproject.toml b/pyproject.toml index a9c5f26..8aaffb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,10 @@ wheel = [ "pkginfo", "packaging", ] +schema = [ + "jsonschema >=4", + "rfc3986-validator", +] dev = [ "pytest", "pytest-cov", @@ -33,6 +37,11 @@ dev = [ # from wheel "pkginfo", "packaging", + # from schema + "jsonschema >=4", + "rfc3986-validator", + # stubs + "types-jsonschema", ] [project.urls] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9c215f7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,25 @@ +import gzip +import shutil +from pathlib import Path + +import pytest + +HERE = Path(__file__).parent +DATA_DIR = Path(__file__).parent / "data" +SPEC_JSON_GZ = sorted(DATA_DIR.glob("*.json.gz")) + + +@pytest.fixture(params=SPEC_JSON_GZ) +def an_historic_spec_gz(request) -> Path: + return request.param + + +@pytest.fixture +def an_historic_spec_json(tmp_path: Path, an_historic_spec_gz: Path) -> Path: + target_path = tmp_path / "pyodide-lock.json" + + with gzip.open(an_historic_spec_gz) as fh_in: + with target_path.open("wb") as fh_out: + shutil.copyfileobj(fh_in, fh_out) + + return target_path diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 0000000..dd5e87c --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,60 @@ +import json +from pathlib import Path +from typing import Any + +import pytest +from jsonschema import ValidationError +from jsonschema.validators import Draft201909Validator + +from pyodide_lock import PyodideLockSpec + +#: a schema that constrains the schema itself for schema syntax +META_SCHEMA = { + "type": "object", + "required": ["description", "$id", "$schema"], + "properties": { + "description": {"type": "string"}, + "$id": {"type": "string", "format": "uri"}, + "$schema": {"type": "string", "format": "uri"}, + "definitions": {"patternProperties": {".*": {"required": ["description"]}}}, + }, +} + +Validator = Draft201909Validator +FORMAT_CHECKER = Draft201909Validator.FORMAT_CHECKER + + +@pytest.fixture +def schema() -> dict[str, Any]: + return PyodideLockSpec.schema() + + +@pytest.fixture +def spec_validator(schema: dict[str, Any]) -> Validator: + return Validator(schema, format_checker=FORMAT_CHECKER) + + +def test_documentation(schema: dict[str, Any]) -> None: + meta_validator = Validator(META_SCHEMA, format_checker=FORMAT_CHECKER) + _assert_validation_errors(meta_validator, schema) + + +def test_validate(an_historic_spec_json: Path, spec_validator: Validator) -> None: + spec_json = json.loads(an_historic_spec_json.read_text(encoding="utf-8")) + _assert_validation_errors(spec_validator, spec_json) + + +def _assert_validation_errors( + validator: Draft201909Validator, + instance: dict[str, Any], + expect_errors: list[str] | None = None, +) -> None: + expect_errors = expect_errors or [] + expect_error_count = len(expect_errors) + + errors: list[ValidationError] = list(validator.iter_errors(instance)) + error_count = len(errors) + + print("\n".join([f"""{err.json_path}: {err.message}""" for err in errors])) + + assert error_count == expect_error_count diff --git a/tests/test_spec.py b/tests/test_spec.py index 7d29ae2..bc72236 100644 --- a/tests/test_spec.py +++ b/tests/test_spec.py @@ -1,5 +1,3 @@ -import gzip -import shutil from copy import deepcopy from pathlib import Path @@ -33,17 +31,10 @@ } -@pytest.mark.parametrize("pyodide_version", ["0.22.1", "0.23.3"]) -def test_lock_spec_parsing(pyodide_version, tmp_path): - source_path = DATA_DIR / f"pyodide-lock-{pyodide_version}.json.gz" - target_path = tmp_path / "pyodide-lock.json" +def test_lock_spec_parsing(an_historic_spec_json: Path, tmp_path): target2_path = tmp_path / "pyodide-lock2.json" - with gzip.open(source_path) as fh_in: - with target_path.open("wb") as fh_out: - shutil.copyfileobj(fh_in, fh_out) - - spec = PyodideLockSpec.from_json(target_path) + spec = PyodideLockSpec.from_json(an_historic_spec_json) spec.to_json(target2_path, indent=2) spec2 = PyodideLockSpec.from_json(target2_path) @@ -53,6 +44,9 @@ def test_lock_spec_parsing(pyodide_version, tmp_path): for key in spec.packages: assert spec.packages[key] == spec2.packages[key] + with pytest.raises(ValueError, match="does not match package version"): + spec.check_wheel_filenames() + def test_check_wheel_filenames(): lock_data = deepcopy(LOCK_EXAMPLE)