From 075289cf826f5036d669af1d76e3275b59ba55e8 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Tue, 18 Jul 2023 21:39:04 +0900 Subject: [PATCH] ENH Support alternative index urls (#74) --- CHANGELOG.md | 7 +- micropip/__init__.py | 2 + micropip/_commands/index_urls.py | 27 ++++++ micropip/_commands/install.py | 17 ++++ micropip/package_index.py | 65 ++++++++++++++ micropip/transaction.py | 19 ++-- pyproject.toml | 1 + tests/conftest.py | 48 +++++++++- .../fake-pkg-micropip-test_json.json.gz | Bin 0 -> 645 bytes .../fake-pkg-micropip-test_simple.json.gz | Bin 0 -> 393 bytes tests/test_install.py | 31 +++++++ tests/test_package_index.py | 85 ++++++++++++++---- tests/test_transaction.py | 1 + 13 files changed, 268 insertions(+), 35 deletions(-) create mode 100644 micropip/_commands/index_urls.py create mode 100644 tests/test_data/pypi_response/fake-pkg-micropip-test_json.json.gz create mode 100644 tests/test_data/pypi_response/fake-pkg-micropip-test_simple.json.gz diff --git a/CHANGELOG.md b/CHANGELOG.md index efe35a4..4bf4bb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `verbose` parameter to micropip.install and micropip.uninstall [#60](https://github.com/pyodide/micropip/pull/60) - +- Added `index_urls` parameter to micropip.install to support installing + from custom package indexes. + [#74](https://github.com/pyodide/micropip/pull/74) +- Added `micropip.set_index_urls` to support installing from custom package + indexes. + [#74](https://github.com/pyodide/micropip/pull/74) ### Fixed - Fixed `micropip.add_mock_package` to work with Pyodide>=0.23.0 diff --git a/micropip/__init__.py b/micropip/__init__.py index 9047f96..3a890d3 100644 --- a/micropip/__init__.py +++ b/micropip/__init__.py @@ -1,4 +1,5 @@ from ._commands.freeze import freeze +from ._commands.index_urls import set_index_urls from ._commands.install import install from ._commands.list import _list as list from ._commands.mock_package import ( @@ -21,5 +22,6 @@ "list_mock_packages", "remove_mock_package", "uninstall", + "set_index_urls", "__version__", ] diff --git a/micropip/_commands/index_urls.py b/micropip/_commands/index_urls.py new file mode 100644 index 0000000..60cc9b1 --- /dev/null +++ b/micropip/_commands/index_urls.py @@ -0,0 +1,27 @@ +from .. import package_index + + +def set_index_urls(urls: list[str] | str) -> None: + """ + Set the index URLs to use when looking up packages. + + - The index URL should support the + `JSON API `__ . + + - The index URL may contain the placeholder {package_name} which will be + replaced with the package name when looking up a package. If it does not + contain the placeholder, the package name will be appended to the URL. + + - If a list of URLs is provided, micropip will try each URL in order until + it finds a package. If no package is found, an error will be raised. + + Parameters + ---------- + urls + A list of URLs or a single URL to use as the package index. + """ + + if isinstance(urls, str): + urls = [urls] + + package_index.INDEX_URLS = urls diff --git a/micropip/_commands/install.py b/micropip/_commands/install.py index a636491..dc856ec 100644 --- a/micropip/_commands/install.py +++ b/micropip/_commands/install.py @@ -16,6 +16,7 @@ async def install( deps: bool = True, credentials: str | None = None, pre: bool = False, + index_urls: list[str] | str | None = None, *, verbose: bool | int = False, ) -> None: @@ -86,6 +87,21 @@ async def install( If ``True``, include pre-release and development versions. By default, micropip only finds stable versions. + index_urls : + + A list of URLs or a single URL to use as the package index when looking + up packages. If None, *https://pypi.org/pypi/{package_name}/json* is used. + + - The index URL should support the + `JSON API `__ . + + - The index URL may contain the placeholder {package_name} which will be + replaced with the package name when looking up a package. If it does not + contain the placeholder, the package name will be appended to the URL. + + - If a list of URLs is provided, micropip will try each URL in order until + it finds a package. If no package is found, an error will be raised. + verbose : Print more information about the process. By default, micropip is silent. Setting ``verbose=True`` will print @@ -117,6 +133,7 @@ async def install( pre=pre, fetch_kwargs=fetch_kwargs, verbose=verbose, + index_urls=index_urls, ) await transaction.gather_requirements(requirements) diff --git a/micropip/package_index.py b/micropip/package_index.py index 0a602cb..c66cbad 100644 --- a/micropip/package_index.py +++ b/micropip/package_index.py @@ -1,3 +1,5 @@ +import json +import string import sys from collections import defaultdict from collections.abc import Generator @@ -7,8 +9,14 @@ from packaging.utils import InvalidWheelFilename from packaging.version import InvalidVersion, Version +from ._compat import fetch_string from ._utils import is_package_compatible, parse_version +DEFAULT_INDEX_URLS = ["https://pypi.org/pypi/{package_name}/json"] +INDEX_URLS = DEFAULT_INDEX_URLS + +_formatter = string.Formatter() + # TODO: Merge this class with WheelInfo @dataclass @@ -182,3 +190,60 @@ def _fast_check_incompatibility(filename: str) -> bool: return False return True + + +def _contain_placeholder(url: str, placeholder: str = "package_name") -> bool: + fields = [parsed[1] for parsed in _formatter.parse(url)] + + return placeholder in fields + + +async def query_package( + name: str, + fetch_kwargs: dict[str, str] | None = None, + index_urls: list[str] | str | None = None, +) -> ProjectInfo: + """ + Query for a package from package indexes. + + Parameters + ---------- + name + Name of the package to search for. + fetch_kwargs + Keyword arguments to pass to the fetch function. + index_urls + A list of URLs or a single URL to use as the package index. + If None, the default index URL is used. + + If a list of URLs is provided, it will be tried in order until + it finds a package. If no package is found, an error will be raised. + """ + global INDEX_URLS + + if not fetch_kwargs: + fetch_kwargs = {} + + if index_urls is None: + index_urls = INDEX_URLS + elif isinstance(index_urls, str): + index_urls = [index_urls] + + for url in index_urls: + if _contain_placeholder(url): + url = url.format(package_name=name) + else: + url = f"{url}/{name}/" + + try: + metadata = await fetch_string(url, fetch_kwargs) + except OSError: + continue + + return ProjectInfo.from_json_api(json.loads(metadata)) + else: + raise ValueError( + f"Can't fetch metadata for '{name}'." + "Please make sure you have entered a correct package name " + "and correctly specified index_urls (if you changed them)." + ) diff --git a/micropip/transaction.py b/micropip/transaction.py index d882d5a..431612b 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -16,10 +16,10 @@ from packaging.utils import canonicalize_name from packaging.version import Version +from . import package_index from ._compat import ( REPODATA_PACKAGES, fetch_bytes, - fetch_string, get_dynlibs, loadDynlib, loadedPackages, @@ -177,6 +177,7 @@ class Transaction: deps: bool pre: bool fetch_kwargs: dict[str, str] + index_urls: list[str] | str | None locked: dict[str, PackageMetadata] = field(default_factory=dict) wheels: list[WheelInfo] = field(default_factory=list) @@ -295,7 +296,9 @@ def eval_marker(e: dict[str, str]) -> bool: ) return - metadata: ProjectInfo = await _get_pypi_json(req.name, self.fetch_kwargs) + metadata = await package_index.query_package( + req.name, self.fetch_kwargs, index_urls=self.index_urls + ) try: wheel = find_wheel(metadata, req) @@ -405,18 +408,6 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo: ) -async def _get_pypi_json(pkgname: str, fetch_kwargs: dict[str, str]) -> ProjectInfo: - url = f"https://pypi.org/pypi/{pkgname}/json" - try: - metadata = await fetch_string(url, fetch_kwargs) - except OSError as e: - raise ValueError( - f"Can't fetch metadata for '{pkgname}' from PyPI. " - "Please make sure you have entered a correct package name." - ) from e - return ProjectInfo.from_json_api(json.loads(metadata)) - - def _generate_package_hash(data: IO[bytes]) -> str: sha256_hash = hashlib.sha256() data.seek(0) diff --git a/pyproject.toml b/pyproject.toml index 20f8f5f..6deaefb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dynamic = ["version"] dependencies = ["packaging>=23.0"] [project.optional-dependencies] test = [ + "pytest-httpserver", "pytest-pyodide", "pytest-cov", "build", diff --git a/tests/conftest.py b/tests/conftest.py index 088193f..153cb39 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +import functools +import gzip import io import sys import zipfile @@ -14,6 +16,12 @@ PLATFORM = f"emscripten_{EMSCRIPTEN_VER.replace('.', '_')}_wasm32" CPVER = f"cp{sys.version_info.major}{sys.version_info.minor}" +TEST_PYPI_RESPONSE_DIR = Path(__file__).parent / "test_data" / "pypi_response" + + +def _read_pypi_response(file: Path) -> bytes: + return gzip.decompress(file.read_bytes()) + def _build(build_dir, dist_dir): import build @@ -186,7 +194,7 @@ def add_pkg_version( self.metadata_map[filename] = metadata self.top_level_map[filename] = top_level - async def _get_pypi_json(self, pkgname, kwargs): + async def query_package(self, pkgname, kwargs, index_urls=None): from micropip.package_index import ProjectInfo try: @@ -229,9 +237,43 @@ def write_file(filename, contents): @pytest.fixture def mock_fetch(monkeypatch, mock_importlib): pytest.importorskip("packaging") - from micropip import transaction + from micropip import package_index, transaction result = mock_fetch_cls() - monkeypatch.setattr(transaction, "_get_pypi_json", result._get_pypi_json) + monkeypatch.setattr(package_index, "query_package", result.query_package) monkeypatch.setattr(transaction, "fetch_bytes", result._fetch_bytes) return result + + +def _mock_package_index_gen( + httpserver, + pkgs=("black", "pytest", "numpy", "pytz", "snowballstemmer"), + content_type="application/json", + suffix="_json.json.gz", +): + # Run a mock server that serves as a package index + import secrets + + base = secrets.token_hex(16) + + for pkg in pkgs: + data = _read_pypi_response(TEST_PYPI_RESPONSE_DIR / f"{pkg}{suffix}") + httpserver.expect_request(f"/{base}/{pkg}/").respond_with_data( + data, + content_type=content_type, + headers={"Access-Control-Allow-Origin": "*"}, + ) + + index_url = httpserver.url_for(base) + + return index_url + + +@pytest.fixture +def mock_package_index_json_api(httpserver): + return functools.partial( + _mock_package_index_gen, + httpserver=httpserver, + suffix="_json.json.gz", + content_type="application/json", + ) diff --git a/tests/test_data/pypi_response/fake-pkg-micropip-test_json.json.gz b/tests/test_data/pypi_response/fake-pkg-micropip-test_json.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..a71c2875cc8260b616d87cd4f7011a5933034b6d GIT binary patch literal 645 zcmV;00($))iwFonL$zc817=}sWi4=PXDw}MV{&hBX>cucWpi|2YIARHE^2dcZUC)S z%aWTg5WM>em$c~7X`LQ36A>JQ zye%nci%1YXdaxRKRd>+~z}bNVb{*BNWDP{o6U+&np}k1$_)OxQm zx85xukf#CPP@_R2uZkKtP~&Bz;ZMUf7*a4i!NL;*tXeeo!hWWyYpUCM_ytYen>E^m zn{bS^RGgtesHEJ$0w5Zhx~(0P)#xNQyBN@ncBhEq@F`5vqzC&WPUTt0t~!p>M6s!U zE-{yN?PdpOW_Fq(r(h@Sva4|Q!J`qGTiqp;jKkPL1RX@)9Yi_@1x~F%wed!^Ng$+v zfQoi=sko0&MGZ0RwA6CQsEbeJM!PGT7m#KVO~ZYe?ha{|?eiqvhf#8%$*xQ@Uya_# z?{`Yp*@Rpw057APOUS7I{aJj;OK(lhHzXSNw<~@~!Z7y0jdHh^WjAT#Fph!<-}Fbc zKPLD-`Kph05oRRWr(tww`(}5Dck$t;_cOPVs;pJV?N&&&K`U+lOMCB4$}GV%G4IcH fSUpcB>80U#$I*;&W*OZ6Z&ufT1ZUa}>jnS-HJCW? literal 0 HcmV?d00001 diff --git a/tests/test_data/pypi_response/fake-pkg-micropip-test_simple.json.gz b/tests/test_data/pypi_response/fake-pkg-micropip-test_simple.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..b7864d5962d7c2c5f14540216bef39aeb4327b5b GIT binary patch literal 393 zcmV;40e1c$iwFpPLbYT717=}sWi4=PXDw}MV{&hBX>cucWpi|2b7^gGY-KKLb8l_{ zrBcgo!ypj6=PM%4#R2??o$u(WRb*}OQlt1mjnm4F^6v%PNo!wvC_(}>%k0d+b+dJY zC|#i?hwbZl^yB*2jRmspkk;5k0X?`v1(n68WsQeup>0ofquRf!eai-FbTCnH#@w9W z@>8lT<_)$9n#xeOJj9nDq~E=ipeZNc>)e(L4I3l=@%?!*~^;r(pCOO+4ttK+cm8T zWLxk49q9ng8H@YlJ+RsJaaxFf-U0vsU3kQq literal 0 HcmV?d00001 diff --git a/tests/test_install.py b/tests/test_install.py index f86ec8c..de6e068 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -368,3 +368,34 @@ async def run_test(selenium, url, name, version): assert f"Successfully installed {name}-{version}" in captured run_test(selenium_standalone_micropip, wheel_url, name, version) + + +@pytest.mark.asyncio +async def test_custom_index_urls(mock_package_index_json_api, monkeypatch): + from io import BytesIO + + mock_server_fake_package = mock_package_index_json_api( + pkgs=["fake-pkg-micropip-test"] + ) + + _wheel_url = "" + + async def _mock_fetch_bytes(url, *args): + nonlocal _wheel_url + _wheel_url = url + return BytesIO(b"fake wheel") + + from micropip import transaction + + monkeypatch.setattr(transaction, "fetch_bytes", _mock_fetch_bytes) + + try: + await micropip.install( + "fake-pkg-micropip-test", index_urls=[mock_server_fake_package] + ) + except Exception: + # We just check that the custom index url was used + # install will fail because the package is not real, but it doesn't matter. + pass + + assert "fake_pkg_micropip_test-1.0.0-py2.py3-none-any.whl" in _wheel_url diff --git a/tests/test_package_index.py b/tests/test_package_index.py index 982feff..03df234 100644 --- a/tests/test_package_index.py +++ b/tests/test_package_index.py @@ -1,25 +1,18 @@ -import gzip import json -from pathlib import Path -from typing import Any import pytest +from conftest import TEST_PYPI_RESPONSE_DIR, _read_pypi_response +import micropip._commands.index_urls as index_urls import micropip.package_index as package_index -TEST_TEMPLATES_DIR = Path(__file__).parent / "test_data" / "pypi_response" - - -def _read_test_data(file: Path) -> dict[str, Any]: - return json.loads(gzip.decompress(file.read_bytes())) - @pytest.mark.parametrize( "name", ["numpy", "black", "pytest", "snowballstemmer", "pytz"] ) def test_project_info_from_json(name): - test_file = TEST_TEMPLATES_DIR / f"{name}_json.json.gz" - test_data = _read_test_data(test_file) + test_file = TEST_PYPI_RESPONSE_DIR / f"{name}_json.json.gz" + test_data = json.loads(_read_pypi_response(test_file)) index = package_index.ProjectInfo.from_json_api(test_data) assert index.name == name @@ -39,8 +32,8 @@ def test_project_info_from_json(name): "name", ["numpy", "black", "pytest", "snowballstemmer", "pytz"] ) def test_project_info_from_simple_json(name): - test_file = TEST_TEMPLATES_DIR / f"{name}_simple.json.gz" - test_data = _read_test_data(test_file) + test_file = TEST_PYPI_RESPONSE_DIR / f"{name}_simple.json.gz" + test_data = json.loads(_read_pypi_response(test_file)) index = package_index.ProjectInfo.from_simple_api(test_data) assert index.name == name @@ -61,11 +54,11 @@ def test_project_info_from_simple_json(name): ) def test_project_info_equal(name): # The different ways of parsing the same data should result in the same - test_file_json = TEST_TEMPLATES_DIR / f"{name}_json.json.gz" - test_file_simple_json = TEST_TEMPLATES_DIR / f"{name}_simple.json.gz" + test_file_json = TEST_PYPI_RESPONSE_DIR / f"{name}_json.json.gz" + test_file_simple_json = TEST_PYPI_RESPONSE_DIR / f"{name}_simple.json.gz" - test_data_json = _read_test_data(test_file_json) - test_data_simple_json = _read_test_data(test_file_simple_json) + test_data_json = json.loads(_read_pypi_response(test_file_json)) + test_data_simple_json = json.loads(_read_pypi_response(test_file_simple_json)) index_json = package_index.ProjectInfo.from_json_api(test_data_json) index_simple_json = package_index.ProjectInfo.from_simple_api(test_data_simple_json) @@ -85,3 +78,61 @@ def test_project_info_equal(name): assert f_json.url == f_simple_json.url assert f_json.version == f_simple_json.version assert f_json.sha256 == f_simple_json.sha256 + + +def test_set_index_urls(): + default_index_urls = package_index.DEFAULT_INDEX_URLS + assert package_index.INDEX_URLS == default_index_urls + + valid_url1 = "https://pkg-index.com/{package_name}/json/" + valid_url2 = "https://another-pkg-index.com/{package_name}" + valid_url3 = "https://another-pkg-index.com/simple/" + try: + index_urls.set_index_urls(valid_url1) + assert package_index.INDEX_URLS == [valid_url1] + + index_urls.set_index_urls([valid_url1, valid_url2, valid_url3]) + assert package_index.INDEX_URLS == [valid_url1, valid_url2, valid_url3] + finally: + index_urls.set_index_urls(default_index_urls) + assert package_index.INDEX_URLS == default_index_urls + + +def test_contain_placeholder(): + assert package_index._contain_placeholder("https://pkg-index.com/{package_name}/") + assert package_index._contain_placeholder( + "https://pkg-index.com/{placeholder}/", placeholder="placeholder" + ) + assert not package_index._contain_placeholder("https://pkg-index.com/") + + +@pytest.mark.asyncio +async def test_query_package(mock_package_index_json_api): + mock_server_snowballstemmer = mock_package_index_json_api(pkgs=["snowballstemmer"]) + mock_server_pytest = mock_package_index_json_api(pkgs=["pytest"]) + + project_info = await package_index.query_package( + "snowballstemmer", index_urls=[mock_server_snowballstemmer] + ) + + assert project_info.name == "snowballstemmer" + assert project_info.releases + + project_info = await package_index.query_package( + "snowballstemmer", index_urls=mock_server_snowballstemmer + ) + + assert project_info.name == "snowballstemmer" + assert project_info.releases + + project_info = await package_index.query_package( + "snowballstemmer", index_urls=[mock_server_pytest, mock_server_snowballstemmer] + ) + + assert project_info.name == "snowballstemmer" + assert project_info.releases + + with pytest.raises(ValueError, match="Can't fetch metadata"): + await package_index.query_package( + "snowballstemmer", index_urls=[mock_server_pytest] + ) diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 3513e56..b22dddf 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -62,6 +62,7 @@ def create_transaction(Transaction): ctx={}, ctx_extras=[], fetch_kwargs={}, + index_urls=None, )