diff --git a/maintainers/scripts/cuda/00-regen-manifest-hashes.sh b/maintainers/scripts/cuda/00-regen-manifest-hashes.sh new file mode 100755 index 000000000000000..cb4774ed764942d --- /dev/null +++ b/maintainers/scripts/cuda/00-regen-manifest-hashes.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env nix-shell +#!nix-shell -I nixpkgs=. -i bash -p "python312.withPackages(ps: with ps; [ cuda-redist-lib ])" +# shellcheck shell=bash + +set -euo pipefail + +trace() { echo >&2 "$@"; } + +trace "Regenerating hashes for CUDA manifest files" + +regen-manifest-hashes \ + --tensorrt-manifest-dir "./pkgs/development/python-modules/cuda-redist-lib/tensorrt" \ + --output "./pkgs/development/cuda-modules/redist-index/data/manifest-hashes.json" + +trace "Done!" \ No newline at end of file diff --git a/maintainers/scripts/cuda/01-copy-manifests.sh b/maintainers/scripts/cuda/01-copy-manifests.sh new file mode 100755 index 000000000000000..05fd0e9f4037bef --- /dev/null +++ b/maintainers/scripts/cuda/01-copy-manifests.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env nix-shell +#!nix-shell -I nixpkgs=. -i bash -p "python312.withPackages(ps: with ps; [ cuda-redist-lib ])" +# shellcheck shell=bash + +set -euo pipefail + +trace() { echo >&2 "$@"; } + +trace "Copying manifest files to tree" + +export TENSORRT_MANIFEST_DIR="./pkgs/development/python-modules/cuda-redist-lib/tensorrt" +export MANIFEST_HASHES_JSON_PATH="./pkgs/development/cuda-modules/redist-index/data/manifest-hashes.json" +regen-manifest-hashes + +trace "Done!" \ No newline at end of file diff --git a/pkgs/by-name/cu/cuda-redist-index/package.nix b/pkgs/by-name/cu/cuda-redist-index/package.nix index a1d61e10bae3be9..78a145001bee2ca 100644 --- a/pkgs/by-name/cu/cuda-redist-index/package.nix +++ b/pkgs/by-name/cu/cuda-redist-index/package.nix @@ -4,6 +4,7 @@ jq, lib, nixVersions, + path, pkgs, srcOnly, writers, @@ -17,7 +18,7 @@ let storePathOf = let - nixpkgsRootInStore = ../../../..; + nixpkgsRootInStore = path; in # NOTE: Since nixpkgsRootInStore is an actual Path, we need to make sure we concatenate the directory separator # with the stringified `path` argument before concatenating it with nixpkgsRootInStore. diff --git a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/__main__.py b/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/__main__.py deleted file mode 100644 index dec1e99fe7f2544..000000000000000 --- a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/__main__.py +++ /dev/null @@ -1,40 +0,0 @@ -import json -import os -from pathlib import Path - -from cuda_redist_lib.index import mk_index - - -def main() -> None: - TENSORRT_MANIFEST_DIR: None | str = os.getenv("TENSORRT_MANIFEST_DIR", None) - if not TENSORRT_MANIFEST_DIR: - raise ValueError("TENSORRT_MANIFEST_DIR must be set") - - with open( - Path(".") - / "pkgs" - / "development" - / "cuda-modules" - / "redist-index" - / "data" - / "indices" - / "sha256-and-relative-path.json", - "w", - encoding="utf-8", - ) as file: - json.dump( - mk_index().model_dump( - by_alias=True, - exclude_none=True, - exclude_unset=True, - mode="json", - ), - file, - indent=2, - sort_keys=True, - ) - file.write("\n") - - -if __name__ == "__main__": - main() diff --git a/pkgs/by-name/cu/cuda-redist-lib/package.nix b/pkgs/by-name/cu/cuda-redist-lib/package.nix deleted file mode 100644 index 3049ce4f6c58bfc..000000000000000 --- a/pkgs/by-name/cu/cuda-redist-lib/package.nix +++ /dev/null @@ -1,88 +0,0 @@ -{ lib, python312Packages }: -let - inherit (lib.fileset) toSource unions; - inherit (lib.trivial) flip importTOML; - callPackage = flip python312Packages.callPackage { }; - pyprojectAttrs = importTOML ./pyproject.toml; -in -callPackage ( - { - annotated-types, - buildPythonPackage, - flit-core, - lib, - makeWrapper, - pydantic, - pyright, - rich, - ruff, - }: - let - finalAttrs = { - pname = pyprojectAttrs.project.name; - inherit (pyprojectAttrs.project) version; - pyproject = true; - src = toSource { - root = ./.; - fileset = unions [ - ./pyproject.toml - ./cuda_redist_lib - ./tensorrt - ]; - }; - nativeBuildInputs = [ makeWrapper ]; - build-system = [ flit-core ]; - dependencies = [ - annotated-types - pydantic - rich - ]; - pythonImportsCheck = [ finalAttrs.pname ]; - nativeCheckInputs = [ - pyright - ruff - ]; - passthru.optional-dependencies.dev = [ - pyright - ruff - ]; - doCheck = true; - checkPhase = - # preCheck - '' - runHook preCheck - '' - # Check with ruff - + '' - echo "Linting with ruff" - ruff check - echo "Checking format with ruff" - ruff format --diff - '' - # Check with pyright - + '' - echo "Typechecking with pyright" - pyright --warnings - echo "Verifying type completeness with pyright" - pyright --verifytypes ${finalAttrs.pname} --ignoreexternal - '' - # postCheck - + '' - runHook postCheck - ''; - postInstall = '' - mkdir -p "$out/share" - cp -r "$src/tensorrt" "$out/share" - wrapProgram "$out/bin/mk-index-of-sha256-and-relative-path" \ - --set TENSORRT_MANIFEST_DIR "$out/share/tensorrt" - ''; - meta = with lib; { - description = pyprojectAttrs.project.description; - homepage = pyprojectAttrs.project.urls.Homepage; - maintainers = with maintainers; [ connorbaker ]; - mainProgram = "mk-index-of-sha256-and-relative-path"; - }; - }; - in - buildPythonPackage finalAttrs -) diff --git a/pkgs/development/cuda-modules/redist-index/data/default.nix b/pkgs/development/cuda-modules/redist-index/data/default.nix index ce11a04486941f3..02a6384539e6e11 100644 --- a/pkgs/development/cuda-modules/redist-index/data/default.nix +++ b/pkgs/development/cuda-modules/redist-index/data/default.nix @@ -1,13 +1,46 @@ -{ lib, ... }: +{ + config, + lib, + pkgs, + ... +}: let + inherit (config.utils) mkRedistURL; + inherit (config.types) + attrs + redistName + sriHash + version + ; inherit (lib.attrsets) mapAttrs; inherit (lib.options) mkOption; - inherit (lib.trivial) const; - inherit (lib.types) nonEmptyListOf nonEmptyStr; + inherit (lib.trivial) const importJSON; + inherit (lib.types) nonEmptyListOf nonEmptyStr package; + inherit (pkgs) fetchurl; in { imports = [ ./indices ]; options.data = mapAttrs (const mkOption) { + manifestHashes = { + description = "Hashes used to retrieve CUDA manifests"; + type = attrs redistName (attrs version sriHash); + default = importJSON ./manifest-hashes.json; + }; + manifests = { + description = "Packages containing CUDA manifests"; + type = attrs redistName (attrs version package); + default = mapAttrs ( + redistName: + mapAttrs ( + version: hash: + fetchurl { + pname = "redistrib-${redistName}"; + url = mkRedistURL redistName "redistrib_${version}.json"; + inherit hash version; + } + ) + ) config.data.manifestHashes; + }; platforms = { description = "List of platforms to use in creation of the platform type."; type = nonEmptyListOf nonEmptyStr; diff --git a/pkgs/development/cuda-modules/redist-index/data/manifest-hashes.json b/pkgs/development/cuda-modules/redist-index/data/manifest-hashes.json new file mode 100644 index 000000000000000..2560a29cdfa2f9d --- /dev/null +++ b/pkgs/development/cuda-modules/redist-index/data/manifest-hashes.json @@ -0,0 +1,116 @@ +{ + "cublasmp": { + "0.1.0": "sha256-s+nqAkhVmNPqQQ7FirTWmXOmbIpoOgZ37Eb2etCZFCg=", + "0.1.1": "sha256-4b3E9cP3kbDC7kQadyhwJZQY0m5lMI7pRplvTDlIkjQ=", + "0.1.2": "sha256-i0v7qLGG6tqrt55aFAQEcEaBptDNm2vRFIKgvPyA3CY=", + "0.2.0": "sha256-f7lCZMPWnexItLfnEFEzsUm1tk37VjU16Gwb2ZUcGoI=", + "0.2.1": "sha256-K9ooauC+KjmWpAewGmgII3SrtAh6yq9BZeiPfFQyKAU=" + }, + "cuda": { + "11.4.4": "sha256-Jnwu3HMUQvjYhgoud3cP5Rd1IX4dFS6qyzrVeu9QgkQ=", + "11.5.2": "sha256-6msVdiGMjzFj+Q6KsjncbKKKH5yru/9dg9ErA2SX8G4=", + "11.6.2": "sha256-7LZJZdZpCtxT8eOD3G3aawVH1uJ59AqvdwHwd0ATr4I=", + "11.7.1": "sha256-0AEkyZWpqfihFYqVgOGlAycYIeRPFWEVwOTtoRqD/MM=", + "11.8.0": "sha256-lBqVCkqzuVMRxQ33s8i8qXPgzdp2/C9LRW0tXk2sAoE=", + "12.0.1": "sha256-FO3+lBlYIS0YT6yXiQ3dLl6wU4P/NgHifzeoUUohX+8=", + "12.1.1": "sha256-uv6jy4Okz1x2Tu7cqsAEDQ08XbP5p0VQ2g57asJNN4w=", + "12.2.2": "sha256-CRy/lO1Z6gcmSqDZMdiKk4DqKferKG635f/RoALEjpo=", + "12.3.2": "sha256-G26s8zXdSYA2M/7VPvJh1iwZPlpW7uUBnn0vY0455+8=", + "12.4.1": "sha256-nNgV87ccLjaG7yIZt3lLgQRPnc76qOIdrPy1vE2TGJI=", + "12.5.1": "sha256-ernHYBSuSQf6G1FzivWZYHpf2Mo6XEu0w7MTOMxkKpM=", + "12.6.1": "sha256-It3+uBpvnO5KcIouO02xw2x9sKHaofM/nH8vEqHnkN4=" + }, + "cudnn": { + "8.5.0": "sha256-AfFFpv0daApkNXenrIMXurCy+I/Sznk/oKmajEgDoGY=", + "8.6.0": "sha256-f29QvtT9ghbcENbvUFdx3A7MmczoE5k6tAXLUHoh1R0=", + "8.7.0": "sha256-J8alYxUuPSI66vIVoMRooWfS0g6FKUoBW2FLynYlUEU=", + "8.8.0": "sha256-AIhlk1xE3NsqiOVSLehBuz1D6IbFt5DxJeVtPobtheA=", + "8.8.1": "sha256-8XdNhGtRZImMql0uMl9EqOtwag1q1+MxSkmHJBfVT6A=", + "8.9.0": "sha256-/l+dhZjrMkA/wBp+9eEr+YOHVUGn9h5FSMtjrAxV9OE=", + "8.9.1": "sha256-QmZni7X1SMHcYHDGiLNEAtBgltFokadAoOwCQcPTgrs=", + "8.9.2": "sha256-4VenfJfNdMn9pq9KPu80v9/u0tvwu12W6jPJmhbZZas=", + "8.9.3": "sha256-Gg7m120TamwiMlAiTCDMB9xjJPit+/HI0rskBa7IKno=", + "8.9.4": "sha256-bQFMFbrv5F8FiLoQLao9CAV4E4wlNgdl/p4qcZCMe3c=", + "8.9.5": "sha256-UiM/1Ws+9J5pwBEi16G8V3jMTDT7vR2/U6QSAibZUsI=", + "8.9.6": "sha256-YGnvkqK5uxjOv7yUSWS9KwJLdvLCw1pDgSmC4LxFzww=", + "8.9.7": "sha256-UoP4ZGd1+YGG4n2ASxngCfBYlmlgxDCoSzRCFfaCiYY=", + "9.0.0": "sha256-QwLm/7iAblblbT3WN6RbgPeX06J05cBhgNWAwgmGlow=", + "9.1.0": "sha256-OfPPF4mkb/ffcsQQLQJA2KhpO+0JN/L5fM9v6PA/pmw=", + "9.1.1": "sha256-0i1WlAXlaD/45WPQDW6MJ+XmqQLFZMI9dSsiqLiz/iA=", + "9.2.0": "sha256-aFLrJ5uV0rV3X3p3N+wTO+0FkQf4Y83YWI865vE+rdc=", + "9.2.1": "sha256-mkGYxZsuZrKxFac26+Tcjz3G14Fhu0lHAvgk2o/He5k=", + "9.3.0": "sha256-0X2aeHg2VzZ1hVApTwPmM6CwI77Ieb8XM0m/s0eBly4=", + "9.4.0": "sha256-buqvxcw9S7Lyg+YpjkxV1MWdfIPF2f2HIaLA5VruTlQ=" + }, + "cudss": { + "0.1.0": "sha256-HU0Xvj6TZVSqjvD4vIH1tCpJi3lw/d91CsUptMM82ys=", + "0.2.0": "sha256-NXkZYTO1oICfsacFN9B8NAxLfW5rwYyBrGnEEdOxxkI=", + "0.2.1": "sha256-ffvibCpTLtPv9G6k1vL+rCVKIhXJ1X/o/+UCPlvXYwM=", + "0.3.0": "sha256-0Kt0ssI0DJvPCsnUGl/HOnOMIspzdPlaaQv2C3MUXFo=" + }, + "cuquantum": { + "0.1.0": "sha256-WgrAfhxrxlPFWSIrWyzGKw6grMCi9CRVdN41EMkr6nU=", + "22.03.0": "sha256-s6lK6dL4WJcNuVdwtYzkiKoKQ+x9yOApgu+rZh5hxZ0=", + "22.05.0": "sha256-L6vSdiXWlrP99rAE+Wi+UGwBMytX/Ba47Opg/MAWvkE=", + "22.07.0": "sha256-KmOBS1Tx3FfDhJ8ioaQgjjYKv0yFg+1bob0g50JLo4c=", + "22.07.1": "sha256-x+mdTlDXWj1q3SgUr0xYQoBG1R4RYHuv1HzxZkKHjb0=", + "22.11.0": "sha256-/YOLr+SrL8FY5ah1pKkt0BbatKTsN/ijhq4JQJpTLyE=", + "23.03.0": "sha256-I6jbPk9FTbF+37jA2kzJlLIQLGloXDWTZTyQWZ01u6k=", + "23.06.0": "sha256-FU54pa7cdR2qOUyfazAkU7OfpYwo9wonkynnJlV/B88=", + "23.06.1": "sha256-kCMPo5oAeFH4xXYAvT/GiLvXh1FVsEi/malu9UNkQic=", + "23.10.0": "sha256-Z/Ljxrn4wrTdLb+4gCW/BFpXcQOg4ucP7N9uLQM1xtc=", + "24.03.0": "sha256-AZaAJ0ciOu82NvI8i2aMnge6fjVXvPcTOMmjwvSsP0Y=", + "24.08.0": "sha256-yPBj5uT0o/r5+KbGs440xfFhBB8f3ccUBnQyr3NSOYw=" + }, + "cusolvermp": { + "0.4.3": "sha256-694dclE+0ABS6mlTOev4NarCrCVDvcfRbmE0TEXsI2U=", + "0.5.0": "sha256-ij4yJk7/oAX84r81qkXrkbcJDiVEPrC1qoJFMuyNTLg=", + "0.5.1": "sha256-UBKMnMJwQDxgNyMzLximMZ9TyjHiLXgjp422Byhm0BA=" + }, + "cusparselt": { + "0.3.0": "sha256-dvsLGpPT0+bmkXEDC3wMsts/sF5mscNnplPf85pYIKg=", + "0.4.0": "sha256-xLtIgS/1e5m6rQzCtowoBh6SnBJigrjemL+YI1ro2X8=", + "0.5.0": "sha256-x0gjsttilp2aXctHzZJ8UeLzaoLU4v/SeQwW9ehdeLI=", + "0.5.1": "sha256-A/7p6Va1mJoPlqhvXmPRtaHv3YYGiEuTyOXNxCw6PKg=", + "0.5.2": "sha256-QMjdBY9leJpDDSj1g6CrE2eU1HNrv49o/1Mf7QfC+PY=", + "0.6.0": "sha256-OAcSeGTYb3YFEnLhm1OA4ctwnO9t1wq3ud1+S7zrOJQ=", + "0.6.1": "sha256-SspjMa/C2fzdZHhvqi0jRMKOfCkGH4d/Rvr6YSxGbdA=", + "0.6.2": "sha256-nQARo8qV3lseb/8gfhBqk2GGNcy34AI15WPvUGRkp3w=" + }, + "cutensor": { + "1.3.2": "sha256-KW1VZmuKz6hVszn9JS4xFsNzmYKQGLfWp1zGHZU7fe4=", + "1.3.3": "sha256-xQjOnoKkk7ouKcve9EgxG1mWML5FxFhTKLBLDsJv7mY=", + "1.4.0": "sha256-Quf+mmD66IXEksHpaJfeCxQLROOs58zzGDHMwHciUpI=", + "1.5.0": "sha256-Tf6sJH5QHFaJqa7WAnsJW9NlVtoAbh4UKrcUli3lOxs=", + "1.6.0": "sha256-4VUxQjeU/wfhgGXX2uXc0C2zAA3zsUJPDoosmKVmZaE=", + "1.6.1": "sha256-9FHh5wln+tOsYr6bUnV7NeQY6+l+TQNjJ4V/yluaO1I=", + "1.6.2": "sha256-Ap0kUNO+/8rFyo2xE5rZLD3f4yX2MTmhM8cEdBzxfoQ=", + "1.7.0": "sha256-jRpT4Gb0lFG8oNEv+6cPd+14rQ2Kp7NpGXh2siMDcC4=", + "2.0.0": "sha256-ASHqLWUN8XVINNjCAWiBEa3o+rqGIi/qCv+iQfQtTBQ=", + "2.0.1": "sha256-MnKEh22Q2h7u8C3fXIJP6wcKf+fAtgoFUeDX78CC4k8=", + "2.0.2.1": "sha256-XeGcpsPt1eYYqwraEbeOQSdxN5Eb9mq5Xv3QBYOApck=" + }, + "nvjpeg2000": { + "0.5.0": "sha256-1SBAKVtY7GdPXQNXshaAu7unPB+XFJq2kGOuB+sxRFA=", + "0.6.0": "sha256-qbRALLjQ/aqfLZx8MjpirNUAaZL/sLEf2zuk2ggWRPo=", + "0.7.0": "sha256-eHzBgwPLfxAAbv45ECV7uSH+3x09kaXBxvPYDITjRfc=", + "0.7.5": "sha256-ZfiHGT8hKk/QOw/YKTJuqIYhz/IXP7lkOR9vwUK5NQs=", + "0.8.0": "sha256-jl2GoSXt+otQAai3yLRHMCrWkB6SFY5eOp+yIVniIZA=" + }, + "nvpl": { + "23.11": "sha256-NLL1MFiYAa7lQHiWPTsbGLcDKP8BCf3QNrRo8umTRm0=", + "24.03": "sha256-sEOima07m9uu2Dly87JosAR/jSnzjXitG2N7FS5SILw=", + "24.7": "sha256-ndYuu5SjHRTAQoITouQqBfMcyCY2decaka/4QfxLNYg=" + }, + "nvtiff": { + "0.3.0": "sha256-SgifVf+g3zSSPYikXvFyxF6HQIHrJ7lsyrNhrlwM4pg=", + "0.3.1": "sha256-gAw9Zvbv09XBY4GDl3Y42SAeWgoeKHqRbSbO+RoMMwE=" + }, + "tensorrt": { + "10.0.1": "sha256-96ZoQQd7CCk9osaTRj8/F6BxRHAYzBFGsgHRuN7Src8=", + "10.1.0": "sha256-nnPqAItOnq3Zu4SHLPVgYvdbker8qISV1jtmR035WOg=", + "10.2.0": "sha256-ZdJ9Ah7esFTkLTyX3Ip7D6BiiiuOOAHnyMUJf9jnRUs=", + "10.3.0": "sha256-GZygJ9FnTpSwHEjz6nuScTeCQNPa52A+vJloQGOOsmI=", + "10.4.0": "sha256-yYEp3AA3PDCN3j4UXcmITd88dfdrS/hZVjoIQ7scuRU=" + } +} diff --git a/pkgs/by-name/cu/cuda-redist-lib/.gitignore b/pkgs/development/python-modules/cuda-redist-lib/.gitignore similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/.gitignore rename to pkgs/development/python-modules/cuda-redist-lib/.gitignore diff --git a/pkgs/by-name/cu/cuda-redist-lib/README.md b/pkgs/development/python-modules/cuda-redist-lib/README.md similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/README.md rename to pkgs/development/python-modules/cuda-redist-lib/README.md diff --git a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/__init__.py b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/__init__.py similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/__init__.py rename to pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/__init__.py diff --git a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/py.typed b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/cmd/__init__.py similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/py.typed rename to pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/cmd/__init__.py diff --git a/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/cmd/mk_index_of_sha256_and_relative_path.py b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/cmd/mk_index_of_sha256_and_relative_path.py new file mode 100644 index 000000000000000..fb7d8b8fea9097f --- /dev/null +++ b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/cmd/mk_index_of_sha256_and_relative_path.py @@ -0,0 +1,55 @@ +import json +from argparse import ArgumentParser, Namespace +from pathlib import Path + +from cuda_redist_lib.index import mk_index + + +def setup_argparse() -> ArgumentParser: + parser = ArgumentParser(description="Generate the index of sha256 and relative path for the cuda redist lib") + _ = parser.add_argument( + "--tensorrt-manifest-dir", + type=Path, + help="The directory of the tensorrt manifests", + required=True, + ) + _ = parser.add_argument( + "--output", + type=Path, + help="The output file", + required=False, + default=Path(".") + / "pkgs" + / "development" + / "cuda-modules" + / "redist-index" + / "data" + / "indices" + / "sha256-and-relative-path.json", + ) + return parser + + +def main() -> None: + parser = setup_argparse() + args: Namespace = parser.parse_args() + tensorrt_manifest_dir: Path = args.tensorrt_manifest_dir + output: Path = args.output + + with open(output, "w", encoding="utf-8") as file: + json.dump( + mk_index(tensorrt_manifest_dir).model_dump( + by_alias=True, + exclude_none=True, + exclude_unset=True, + mode="json", + ), + file, + indent=2, + sort_keys=True, + ) + file.write("\n") + + +if __name__ == "__main__": + main() diff --git a/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/cmd/regen_manifest_hashes.py b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/cmd/regen_manifest_hashes.py new file mode 100644 index 000000000000000..15c0b1df3e467b3 --- /dev/null +++ b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/cmd/regen_manifest_hashes.py @@ -0,0 +1,52 @@ +import json +from argparse import ArgumentParser, Namespace +from pathlib import Path + +from cuda_redist_lib.extra_types import RedistNames +from cuda_redist_lib.manifest import ( + get_nvidia_manifest_str, + get_nvidia_manifest_versions, +) +from cuda_redist_lib.utilities import mk_sri_hash + + +def setup_argparse() -> ArgumentParser: + parser = ArgumentParser(description="Generate the index of SRI hashes for each CUDA redistributable manifest") + _ = parser.add_argument( + "--tensorrt-manifest-dir", + type=Path, + help="The directory of the tensorrt manifests", + required=True, + ) + _ = parser.add_argument( + "--output", + type=Path, + help="The output file", + required=False, + default=Path(".") / "pkgs" / "development" / "cuda-modules" / "redist-index" / "data" / "manifest-hashes.json", + ) + return parser + + +def main() -> None: + parser = setup_argparse() + args: Namespace = parser.parse_args() + tensorrt_manifest_dir: Path = args.tensorrt_manifest_dir + output: Path = args.output + + d = { + redist_name: { + version: mk_sri_hash(get_nvidia_manifest_str(redist_name, version, tensorrt_manifest_dir)) + for version in get_nvidia_manifest_versions(redist_name, tensorrt_manifest_dir) + } + for redist_name in RedistNames + } + + with open(output, mode="w", encoding="utf-8") as file: + json.dump( + d, + file, + indent=2, + sort_keys=True, + ) + file.write("\n") diff --git a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/extra_pydantic.py b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/extra_pydantic.py similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/extra_pydantic.py rename to pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/extra_pydantic.py diff --git a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/extra_types.py b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/extra_types.py similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/extra_types.py rename to pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/extra_types.py diff --git a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/index.py b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/index.py similarity index 84% rename from pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/index.py rename to pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/index.py index a4ecb7662e21b08..0ff5dcaec2736cf 100644 --- a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/index.py +++ b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/index.py @@ -134,18 +134,33 @@ def mk_package_hashes( if actual_relative_path != expected_relative_path: # TensorRT will fail this check because it doesn't follow the usual naming convention. if release_info.name != "NVIDIA TensorRT": - logger.info("Expected relative path to be %s, got %s", expected_relative_path, actual_relative_path) - package_info = PackageInfo.model_validate({"sha256": sha256, "relative_path": actual_relative_path}) + logger.info( + "Expected relative path to be %s, got %s", + expected_relative_path, + actual_relative_path, + ) + package_info = PackageInfo.model_validate({ + "sha256": sha256, + "relative_path": actual_relative_path, + }) else: - package_info = PackageInfo.model_validate({"sha256": sha256, "relative_path": None}) + package_info = PackageInfo.model_validate({ + "sha256": sha256, + "relative_path": None, + }) infos[cuda_variant_name] = package_info return PackageVariants.model_validate(infos) -def mk_manifest(redist_name: RedistName, version: Version, nvidia_manifest: None | NvidiaManifest = None) -> Manifest: +def mk_manifest( + redist_name: RedistName, + version: Version, + nvidia_manifest: None | NvidiaManifest = None, + tensorrt_manifest_dir: None | Path = None, +) -> Manifest: if nvidia_manifest is None: - nvidia_manifest = get_nvidia_manifest(redist_name, version) + nvidia_manifest = get_nvidia_manifest(redist_name, version, tensorrt_manifest_dir) releases: dict[str, Release] = { package_name: release @@ -157,10 +172,11 @@ def mk_manifest(redist_name: RedistName, version: Version, nvidia_manifest: None return Manifest.model_validate(releases) -def mk_index() -> Index: +def mk_index(tensorrt_manifest_dir: Path) -> Index: return Index.model_validate({ redist_name: { - version: mk_manifest(redist_name, version) for version in get_nvidia_manifest_versions(redist_name) + version: mk_manifest(redist_name, version, tensorrt_manifest_dir=tensorrt_manifest_dir) + for version in get_nvidia_manifest_versions(redist_name, tensorrt_manifest_dir) } for redist_name in RedistNames }) diff --git a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/logger.py b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/logger.py similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/logger.py rename to pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/logger.py diff --git a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/manifest.py b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/manifest.py similarity index 85% rename from pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/manifest.py rename to pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/manifest.py index cc919f33605b89a..b11a9052aec6790 100644 --- a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/manifest.py +++ b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/manifest.py @@ -1,9 +1,7 @@ # NOTE: Open bugs in Pydantic like https://github.com/pydantic/pydantic/issues/8984 prevent the full switch to the type # keyword introduced in Python 3.12. -import os import re from collections.abc import Mapping, Sequence -from io import TextIOWrapper from logging import Logger from pathlib import Path from typing import Annotated, Final, Self, override @@ -37,16 +35,24 @@ class NvidiaPackage(PydanticObject): relative_path: Annotated[Path, Field(description="Relative path to the package from the index URL.")] sha256: Annotated[Sha256, Field(description="SHA256 hash of the package.")] md5: Annotated[Md5, Field(description="MD5 hash of the package.")] - size: Annotated[str, Field(description="Size of the package in bytes, as a string.", pattern=r"\d+")] + size: Annotated[ + str, + Field(description="Size of the package in bytes, as a string.", pattern=r"\d+"), + ] class NvidiaReleaseCommon(PydanticObject): name: Annotated[str, Field(description="Full name and description of the release.")] license: Annotated[str, Field(description="License under which the release is distributed.")] - license_path: Annotated[None | Path, Field(description="Relative path to the license file.", default=None)] + license_path: Annotated[ + None | Path, + Field(description="Relative path to the license file.", default=None), + ] version: Annotated[Version, Field(description="Version of the release.")] - def packages(self) -> Mapping[RedistPlatform, NvidiaPackage | Mapping[CudaVariant, NvidiaPackage]]: + def packages( + self, + ) -> Mapping[RedistPlatform, NvidiaPackage | Mapping[CudaVariant, NvidiaPackage]]: raise NotImplementedError() @@ -98,7 +104,10 @@ class NvidiaReleaseV3(NvidiaReleaseCommon): NvidiaPackage | Mapping[CudaVariant, NvidiaPackage], # NOTE: `source` does not use cuda variants ] - cuda_variant: Annotated[Sequence[MajorVersion], Field(description="CUDA variants supported by the release.")] + cuda_variant: Annotated[ + Sequence[MajorVersion], + Field(description="CUDA variants supported by the release."), + ] @field_validator("cuda_variant", mode="after") @classmethod @@ -147,7 +156,9 @@ def check_extra_have_cuda_variant_keys(self) -> Self: return self @override - def packages(self) -> Mapping[RedistPlatform, NvidiaPackage | Mapping[CudaVariant, NvidiaPackage]]: + def packages( + self, + ) -> Mapping[RedistPlatform, NvidiaPackage | Mapping[CudaVariant, NvidiaPackage]]: return self.__pydantic_extra__ @@ -211,7 +222,9 @@ def is_ignored_nvidia_manifest(redist_name: RedistName, version: Version) -> Non return None -def get_nvidia_manifest_versions(redist_name: RedistName) -> Sequence[Version]: +def get_nvidia_manifest_versions( + redist_name: RedistName, tensorrt_manifest_dir: None | Path = None +) -> Sequence[Version]: logger.info("Getting versions for %s", redist_name) regex_pattern = re.compile( r""" @@ -239,10 +252,10 @@ def get_nvidia_manifest_versions(redist_name: RedistName) -> Sequence[Version]: listing: str match redist_name: case "tensorrt": + if tensorrt_manifest_dir is None: + raise ValueError("Must provide the path to the tensorrt manifests") listing = "\n".join( - redistrib_path.name - for redistrib_path in Path(os.environ["TENSORRT_MANIFEST_DIR"]).iterdir() - if redistrib_path.is_file() + redistrib_path.name for redistrib_path in tensorrt_manifest_dir.iterdir() if redistrib_path.is_file() ) case _: with request.urlopen(f"{RedistUrlPrefix}/{redist_name}/redist/index.html") as response: @@ -266,13 +279,20 @@ def get_nvidia_manifest_versions(redist_name: RedistName) -> Sequence[Version]: return [version for _, version in version_dict.values()] -def get_nvidia_manifest(redist_name: RedistName, version: Version) -> NvidiaManifest: +def get_nvidia_manifest_str( + redist_name: RedistName, version: Version, tensorrt_manifest_dir: None | Path = None +) -> bytes: logger.info("Getting manifest for %s %s", redist_name, version) - context: TextIOWrapper match redist_name: case "tensorrt": - context = (Path(os.environ["TENSORRT_MANIFEST_DIR"]) / f"redistrib_{version}.json").open(encoding="utf-8") + if tensorrt_manifest_dir is None: + raise ValueError("Must provide the path to the tensorrt manifests") + return (tensorrt_manifest_dir / f"redistrib_{version}.json").read_bytes() case _: - context = request.urlopen(f"{RedistUrlPrefix}/{redist_name}/redist/redistrib_{version}.json") + return request.urlopen(f"{RedistUrlPrefix}/{redist_name}/redist/redistrib_{version}.json").read() + - return NvidiaManifest.model_validate_json(context.read()) +def get_nvidia_manifest( + redist_name: RedistName, version: Version, tensorrt_manifest_dir: None | Path = None +) -> NvidiaManifest: + return NvidiaManifest.model_validate_json(get_nvidia_manifest_str(redist_name, version, tensorrt_manifest_dir)) diff --git a/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/py.typed b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/py.typed new file mode 100644 index 000000000000000..e69de29bb2d1d64 diff --git a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/utilities.py b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/utilities.py similarity index 75% rename from pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/utilities.py rename to pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/utilities.py index 5197e2f5d0cf5c3..d318f367fe5d7e4 100644 --- a/pkgs/by-name/cu/cuda-redist-lib/cuda_redist_lib/utilities.py +++ b/pkgs/development/python-modules/cuda-redist-lib/cuda_redist_lib/utilities.py @@ -1,19 +1,30 @@ # NOTE: Open bugs in Pydantic like https://github.com/pydantic/pydantic/issues/8984 prevent the full switch to the type # keyword introduced in Python 3.12. import base64 +from hashlib import sha256 from pathlib import Path from cuda_redist_lib.extra_types import CudaVariant, PackageName, RedistPlatform, Sha256, SriHash, SriHashTA, Version +def sha256_bytes_to_sri_hash(sha256_bytes: bytes) -> SriHash: + base64_hash = base64.b64encode(sha256_bytes).decode("utf-8") + sri_hash = f"sha256-{base64_hash}" + return SriHashTA.validate_strings(sri_hash) + + def sha256_to_sri_hash(sha256: Sha256) -> SriHash: """ Convert a Base16 SHA-256 hash to a Subresource Integrity (SRI) hash. """ - sha256_bytes = bytes.fromhex(sha256) - base64_hash = base64.b64encode(sha256_bytes).decode("utf-8") - sri_hash = f"sha256-{base64_hash}" - return SriHashTA.validate_strings(sri_hash) + return sha256_bytes_to_sri_hash(bytes.fromhex(sha256)) + + +def mk_sri_hash(bs: bytes) -> SriHash: + """ + Compute a Subresource Integrity (SRI) hash from a byte string. + """ + return sha256_bytes_to_sri_hash(sha256(bs).digest()) def mk_relative_path( diff --git a/pkgs/development/python-modules/cuda-redist-lib/default.nix b/pkgs/development/python-modules/cuda-redist-lib/default.nix new file mode 100644 index 000000000000000..22900aae4c10d9f --- /dev/null +++ b/pkgs/development/python-modules/cuda-redist-lib/default.nix @@ -0,0 +1,76 @@ +{ + annotated-types, + buildPythonPackage, + flit-core, + lib, + makeWrapper, + pydantic, + pyright, + pythonOlder, + rich, + ruff, +}: +let + inherit (lib.fileset) toSource unions; + inherit (lib.trivial) importTOML; + pyprojectAttrs = importTOML ./pyproject.toml; + finalAttrs = { + pname = pyprojectAttrs.project.name; + inherit (pyprojectAttrs.project) version; + pyproject = true; + disabled = pythonOlder "3.12"; + src = toSource { + root = ./.; + fileset = unions [ + ./pyproject.toml + ./cuda_redist_lib + ]; + }; + nativeBuildInputs = [ makeWrapper ]; + build-system = [ flit-core ]; + dependencies = [ + annotated-types + pydantic + rich + ]; + pythonImportsCheck = [ finalAttrs.pname ]; + nativeCheckInputs = [ + pyright + ruff + ]; + passthru.optional-dependencies.dev = [ + pyright + ruff + ]; + doCheck = true; + checkPhase = + # preCheck + '' + runHook preCheck + '' + # Check with ruff + + '' + echo "Linting with ruff" + ruff check + echo "Checking format with ruff" + ruff format --diff + '' + # Check with pyright + + '' + echo "Typechecking with pyright" + pyright --warnings + echo "Verifying type completeness with pyright" + pyright --verifytypes ${finalAttrs.pname} --ignoreexternal + '' + # postCheck + + '' + runHook postCheck + ''; + meta = with lib; { + description = pyprojectAttrs.project.description; + homepage = pyprojectAttrs.project.urls.Homepage; + maintainers = with maintainers; [ connorbaker ]; + }; + }; +in +buildPythonPackage finalAttrs diff --git a/pkgs/by-name/cu/cuda-redist-lib/pyproject.toml b/pkgs/development/python-modules/cuda-redist-lib/pyproject.toml similarity index 75% rename from pkgs/by-name/cu/cuda-redist-lib/pyproject.toml rename to pkgs/development/python-modules/cuda-redist-lib/pyproject.toml index f502ed1913059a3..c69fda5a025ba95 100644 --- a/pkgs/by-name/cu/cuda-redist-lib/pyproject.toml +++ b/pkgs/development/python-modules/cuda-redist-lib/pyproject.toml @@ -15,10 +15,11 @@ dependencies = ["annotated-types", "pydantic", "rich"] dev = ["ruff"] [project.scripts] -mk-index-of-sha256-and-relative-path = "cuda_redist_lib:__main__.main" +regen-manifest-hashes = "cuda_redist_lib.cmd:regen_manifest_hashes.main" +mk-index-of-sha256-and-relative-path = "cuda_redist_lib.cmd:mk_index_of_sha256_and_relative_path.main" [project.urls] -Homepage = "https://github.com/NixOS/nixpkgs/tree/master/pkgs/by-name/cu/cuda-redist-lib" +Homepage = "https://github.com/NixOS/nixpkgs/tree/master/pkgs/development/python-modules/cuda-redist-lib" [tool.ruff] line-length = 120 diff --git a/pkgs/by-name/cu/cuda-redist-lib/tensorrt/README.md b/pkgs/development/python-modules/cuda-redist-lib/tensorrt/README.md similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/tensorrt/README.md rename to pkgs/development/python-modules/cuda-redist-lib/tensorrt/README.md diff --git a/pkgs/by-name/cu/cuda-redist-lib/tensorrt/helper.sh b/pkgs/development/python-modules/cuda-redist-lib/tensorrt/helper.sh similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/tensorrt/helper.sh rename to pkgs/development/python-modules/cuda-redist-lib/tensorrt/helper.sh diff --git a/pkgs/by-name/cu/cuda-redist-lib/tensorrt/redistrib_10.0.1.json b/pkgs/development/python-modules/cuda-redist-lib/tensorrt/redistrib_10.0.1.json similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/tensorrt/redistrib_10.0.1.json rename to pkgs/development/python-modules/cuda-redist-lib/tensorrt/redistrib_10.0.1.json diff --git a/pkgs/by-name/cu/cuda-redist-lib/tensorrt/redistrib_10.1.0.json b/pkgs/development/python-modules/cuda-redist-lib/tensorrt/redistrib_10.1.0.json similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/tensorrt/redistrib_10.1.0.json rename to pkgs/development/python-modules/cuda-redist-lib/tensorrt/redistrib_10.1.0.json diff --git a/pkgs/by-name/cu/cuda-redist-lib/tensorrt/redistrib_10.2.0.json b/pkgs/development/python-modules/cuda-redist-lib/tensorrt/redistrib_10.2.0.json similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/tensorrt/redistrib_10.2.0.json rename to pkgs/development/python-modules/cuda-redist-lib/tensorrt/redistrib_10.2.0.json diff --git a/pkgs/by-name/cu/cuda-redist-lib/tensorrt/redistrib_10.3.0.json b/pkgs/development/python-modules/cuda-redist-lib/tensorrt/redistrib_10.3.0.json similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/tensorrt/redistrib_10.3.0.json rename to pkgs/development/python-modules/cuda-redist-lib/tensorrt/redistrib_10.3.0.json diff --git a/pkgs/by-name/cu/cuda-redist-lib/tensorrt/redistrib_10.4.0.json b/pkgs/development/python-modules/cuda-redist-lib/tensorrt/redistrib_10.4.0.json similarity index 100% rename from pkgs/by-name/cu/cuda-redist-lib/tensorrt/redistrib_10.4.0.json rename to pkgs/development/python-modules/cuda-redist-lib/tensorrt/redistrib_10.4.0.json diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index df5d546d1964e7f..46d01c343fb79a8 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2704,6 +2704,8 @@ self: super: with self; { cu2qu = callPackage ../development/python-modules/cu2qu { }; + cuda-redist-lib = callPackage ../development/python-modules/cuda-redist-lib { }; + customtkinter = callPackage ../development/python-modules/customtkinter { }; cucumber-tag-expressions = callPackage ../development/python-modules/cucumber-tag-expressions { };