diff options
Diffstat (limited to 'nixpkgs/pkgs/development/interpreters/python')
10 files changed, 167 insertions, 20 deletions
diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/default.nix b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/default.nix index 5cdd307e70fe..a77206ae3852 100644 --- a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/default.nix +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/default.nix @@ -344,7 +344,7 @@ in with passthru; stdenv.mkDerivation ({ ''; license = lib.licenses.psfl; platforms = lib.platforms.all; - maintainers = with lib.maintainers; [ fridh thiagokokada ]; + maintainers = with lib.maintainers; [ fridh ]; knownVulnerabilities = [ "Python 2.7 has reached its end of life after 2020-01-01. See https://www.python.org/doc/sunset-python-2/." # Quote: That means that we will not improve it anymore after that day, diff --git a/nixpkgs/pkgs/development/interpreters/python/default.nix b/nixpkgs/pkgs/development/interpreters/python/default.nix index 6a201b066f8f..2674971670fe 100644 --- a/nixpkgs/pkgs/development/interpreters/python/default.nix +++ b/nixpkgs/pkgs/development/interpreters/python/default.nix @@ -30,10 +30,10 @@ sourceVersion = { major = "3"; minor = "11"; - patch = "6"; + patch = "7"; suffix = ""; }; - hash = "sha256-D6t4+n8TP084IQxiYNkNfA1ccZhEZBnOBX7HrC5vXzg="; + hash = "sha256-GOGqfmb/OlhCPVntIoFaaVTlM0ISLEXfIMlod8Biubc="; }; }; diff --git a/nixpkgs/pkgs/development/interpreters/python/hooks/default.nix b/nixpkgs/pkgs/development/interpreters/python/hooks/default.nix index d06f3db334da..0557c62eeff4 100644 --- a/nixpkgs/pkgs/development/interpreters/python/hooks/default.nix +++ b/nixpkgs/pkgs/development/interpreters/python/hooks/default.nix @@ -173,6 +173,16 @@ in { }; } ./python-remove-tests-dir-hook.sh) {}; + pythonRuntimeDepsCheckHook = callPackage ({ makePythonHook, packaging }: + makePythonHook { + name = "python-runtime-deps-check-hook.sh"; + propagatedBuildInputs = [ packaging ]; + substitutions = { + inherit pythonInterpreter pythonSitePackages; + hook = ./python-runtime-deps-check-hook.py; + }; + } ./python-runtime-deps-check-hook.sh) {}; + setuptoolsBuildHook = callPackage ({ makePythonHook, setuptools, wheel }: makePythonHook { name = "setuptools-setup-hook"; diff --git a/nixpkgs/pkgs/development/interpreters/python/hooks/python-relax-deps-hook.sh b/nixpkgs/pkgs/development/interpreters/python/hooks/python-relax-deps-hook.sh index 1ac91fb40e4e..293bd5cebd50 100644 --- a/nixpkgs/pkgs/development/interpreters/python/hooks/python-relax-deps-hook.sh +++ b/nixpkgs/pkgs/development/interpreters/python/hooks/python-relax-deps-hook.sh @@ -52,7 +52,7 @@ _pythonRelaxDeps() { else for dep in $pythonRelaxDeps; do sed -i "$metadata_file" -r \ - -e "s/(Requires-Dist: $dep\s*(\[[^]]+\])?)[^;]*(;.*)?/\1\3/" + -e "s/(Requires-Dist: $dep\s*(\[[^]]+\])?)[^;]*(;.*)?/\1\3/i" done fi } diff --git a/nixpkgs/pkgs/development/interpreters/python/hooks/python-runtime-deps-check-hook.py b/nixpkgs/pkgs/development/interpreters/python/hooks/python-runtime-deps-check-hook.py new file mode 100644 index 000000000000..5a3a91939175 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/hooks/python-runtime-deps-check-hook.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +""" +The runtimeDependenciesHook validates, that all dependencies specified +in wheel metadata are available in the local environment. + +In case that does not hold, it will print missing dependencies and +violated version constraints. +""" + + +import importlib.metadata +import re +import sys +import tempfile +from argparse import ArgumentParser +from zipfile import ZipFile + +from packaging.metadata import Metadata, parse_email +from packaging.requirements import Requirement + +argparser = ArgumentParser() +argparser.add_argument("wheel", help="Path to the .whl file to test") + + +def error(msg: str) -> None: + print(f" - {msg}", file=sys.stderr) + + +def normalize_name(name: str) -> str: + """ + Normalize package names according to PEP503 + """ + return re.sub(r"[-_.]+", "-", name).lower() + + +def get_manifest_text_from_wheel(wheel: str) -> str: + """ + Given a path to a wheel, this function will try to extract the + METADATA file in the wheels .dist-info directory. + """ + with ZipFile(wheel) as zipfile: + for zipinfo in zipfile.infolist(): + if zipinfo.filename.endswith(".dist-info/METADATA"): + with tempfile.TemporaryDirectory() as tmp: + path = zipfile.extract(zipinfo, path=tmp) + with open(path, encoding="utf-8") as fd: + return fd.read() + + raise RuntimeError("No METADATA file found in wheel") + + +def get_metadata(wheel: str) -> Metadata: + """ + Given a path to a wheel, returns a parsed Metadata object. + """ + text = get_manifest_text_from_wheel(wheel) + raw, _ = parse_email(text) + metadata = Metadata.from_raw(raw) + + return metadata + + +def test_requirement(requirement: Requirement) -> bool: + """ + Given a requirement specification, tests whether the dependency can + be resolved in the local environment, and whether it satisfies the + specified version constraints. + """ + if requirement.marker and not requirement.marker.evaluate(): + # ignore requirements with incompatible markers + return True + + package_name = normalize_name(requirement.name) + + try: + package = importlib.metadata.distribution(requirement.name) + except importlib.metadata.PackageNotFoundError: + error(f"{package_name} not installed") + return False + + if package.version not in requirement.specifier: + error( + f"{package_name}{requirement.specifier} not satisfied by version {package.version}" + ) + return False + + return True + + +if __name__ == "__main__": + args = argparser.parse_args() + + metadata = get_metadata(args.wheel) + tests = [test_requirement(requirement) for requirement in metadata.requires_dist] + + if not all(tests): + sys.exit(1) diff --git a/nixpkgs/pkgs/development/interpreters/python/hooks/python-runtime-deps-check-hook.sh b/nixpkgs/pkgs/development/interpreters/python/hooks/python-runtime-deps-check-hook.sh new file mode 100644 index 000000000000..43a2f9b88745 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/hooks/python-runtime-deps-check-hook.sh @@ -0,0 +1,20 @@ +# Setup hook for PyPA installer. +echo "Sourcing python-runtime-deps-check-hook" + +pythonRuntimeDepsCheckHook() { + echo "Executing pythonRuntimeDepsCheck" + + export PYTHONPATH="$out/@pythonSitePackages@:$PYTHONPATH" + + for wheel in dist/*.whl; do + echo "Checking runtime dependencies for $(basename $wheel)" + @pythonInterpreter@ @hook@ "$wheel" + done + + echo "Finished executing pythonRuntimeDepsCheck" +} + +if [ -z "${dontCheckRuntimeDeps-}" ]; then + echo "Using pythonRuntimeDepsCheckHook" + preInstallPhases+=" pythonRuntimeDepsCheckHook" +fi diff --git a/nixpkgs/pkgs/development/interpreters/python/mk-python-derivation.nix b/nixpkgs/pkgs/development/interpreters/python/mk-python-derivation.nix index c14c6bc096fd..e6f9087de866 100644 --- a/nixpkgs/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/nixpkgs/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -19,6 +19,7 @@ , pythonOutputDistHook , pythonRemoveBinBytecodeHook , pythonRemoveTestsDirHook +, pythonRuntimeDepsCheckHook , setuptoolsBuildHook , setuptoolsCheckHook , wheelUnpackHook @@ -229,6 +230,13 @@ let } else pypaBuildHook + ) ( + if isBootstrapPackage then + pythonRuntimeDepsCheckHook.override { + inherit (python.pythonOnBuildForHost.pkgs.bootstrap) packaging; + } + else + pythonRuntimeDepsCheckHook )] ++ lib.optionals (format' == "wheel") [ wheelUnpackHook ] ++ lib.optionals (format' == "egg") [ diff --git a/nixpkgs/pkgs/development/interpreters/python/pypy/default.nix b/nixpkgs/pkgs/development/interpreters/python/pypy/default.nix index 84af77bade9e..c64c65df350e 100644 --- a/nixpkgs/pkgs/development/interpreters/python/pypy/default.nix +++ b/nixpkgs/pkgs/development/interpreters/python/pypy/default.nix @@ -1,9 +1,12 @@ { lib, stdenv, substituteAll, fetchurl -, zlib ? null, zlibSupport ? true, bzip2, pkg-config, libffi, libunwind, Security +, zlibSupport ? true, zlib +, bzip2, pkg-config, libffi, libunwind, Security , sqlite, openssl, ncurses, python, expat, tcl, tk, tix, libX11 -, self, gdbm, db, xz -, python-setup-hook +, gdbm, db, xz, python-setup-hook +, optimizationLevel ? "jit", boehmgc # For the Python package set +, hash +, self , packageOverrides ? (self: super: {}) , pkgsBuildBuild , pkgsBuildHost @@ -12,7 +15,6 @@ , pkgsTargetTarget , sourceVersion , pythonVersion -, hash , passthruFun , pythonAttr ? "pypy${lib.substring 0 1 pythonVersion}${lib.substring 2 3 pythonVersion}" }: @@ -59,6 +61,8 @@ in with passthru; stdenv.mkDerivation rec { stdenv.cc.libc ] ++ lib.optionals zlibSupport [ zlib + ] ++ lib.optionals (lib.any (l: l == optimizationLevel) [ "0" "1" "2" "3"]) [ + boehmgc ] ++ lib.optionals stdenv.isDarwin [ libunwind Security ]; @@ -102,7 +106,7 @@ in with passthru; stdenv.mkDerivation rec { ${pythonForPypy.interpreter} rpython/bin/rpython \ --make-jobs="$NIX_BUILD_CORES" \ - -Ojit \ + -O${optimizationLevel} \ --batch pypy/goal/targetpypystandalone.py runHook postBuild @@ -195,10 +199,11 @@ in with passthru; stdenv.mkDerivation rec { enableParallelBuilding = true; # almost no parallelization without STM meta = with lib; { - homepage = "http://pypy.org/"; + homepage = "https://www.pypy.org/"; description = "Fast, compliant alternative implementation of the Python language (${pythonVersion})"; license = licenses.mit; platforms = [ "aarch64-linux" "x86_64-linux" "aarch64-darwin" "x86_64-darwin" ]; + broken = optimizationLevel == "0"; # generates invalid code maintainers = with maintainers; [ andersk ]; }; } diff --git a/nixpkgs/pkgs/development/interpreters/python/tests.nix b/nixpkgs/pkgs/development/interpreters/python/tests.nix index 67670ceb6546..df4484f9ec68 100644 --- a/nixpkgs/pkgs/development/interpreters/python/tests.nix +++ b/nixpkgs/pkgs/development/interpreters/python/tests.nix @@ -125,6 +125,9 @@ let extension = self: super: { foobar = super.numpy; }; + # `pythonInterpreters.pypy39_prebuilt` does not expose an attribute + # name (is not present in top-level `pkgs`). + is_prebuilt = python: python.pythonAttr == null; in lib.optionalAttrs (python.isPy3k) ({ test-packageOverrides = let myPython = let @@ -138,7 +141,10 @@ let # test-overrideScope = let # myPackages = python.pkgs.overrideScope extension; # in assert myPackages.foobar == myPackages.numpy; myPackages.python.withPackages(ps: with ps; [ foobar ]); - } // lib.optionalAttrs (python ? pythonAttr) { + # + # Have to skip prebuilt python as it's not present in top-level + # `pkgs` as an attribute. + } // lib.optionalAttrs (python ? pythonAttr && !is_prebuilt python) { # Test applying overrides using pythonPackagesOverlays. test-pythonPackagesExtensions = let pkgs_ = pkgs.extend(final: prev: { diff --git a/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py b/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py index d9b539926b83..9e46a11141a6 100755 --- a/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py +++ b/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py @@ -210,7 +210,7 @@ def _determine_latest_version(current_version, target, versions): return (max(sorted(versions))).raw_version -def _get_latest_version_pypi(package, extension, current_version, target): +def _get_latest_version_pypi(attr_path, package, extension, current_version, target): """Get latest version and hash from PyPI.""" url = "{}/{}/json".format(INDEX, package) json = _fetch_page(url) @@ -234,7 +234,7 @@ def _get_latest_version_pypi(package, extension, current_version, target): return version, sha256, None -def _get_latest_version_github(package, extension, current_version, target): +def _get_latest_version_github(attr_path, package, extension, current_version, target): def strip_prefix(tag): return re.sub("^[^0-9]*", "", tag) @@ -242,9 +242,6 @@ def _get_latest_version_github(package, extension, current_version, target): matches = re.findall(r"^([^0-9]*)", string) return next(iter(matches), "") - # when invoked as an updateScript, UPDATE_NIX_ATTR_PATH will be set - # this allows us to work with packages which live outside of python-modules - attr_path = os.environ.get("UPDATE_NIX_ATTR_PATH", f"python3Packages.{package}") try: homepage = subprocess.check_output( [ @@ -421,13 +418,17 @@ def _update_package(path, target): # Attempt a fetch using each pname, e.g. backports-zoneinfo vs backports.zoneinfo successful_fetch = False for pname in pnames: - if BULK_UPDATE and _skip_bulk_update(f"python3Packages.{pname}"): + # when invoked as an updateScript, UPDATE_NIX_ATTR_PATH will be set + # this allows us to work with packages which live outside of python-modules + attr_path = os.environ.get("UPDATE_NIX_ATTR_PATH", f"python3Packages.{pname}") + + if BULK_UPDATE and _skip_bulk_update(attr_path): raise ValueError(f"Bulk update skipped for {pname}") - elif _get_attr_value(f"python3Packages.{pname}.cargoDeps") is not None: + elif _get_attr_value(f"{attr_path}.cargoDeps") is not None: raise ValueError(f"Cargo dependencies are unsupported, skipping {pname}") try: new_version, new_sha256, prefix = FETCHERS[fetcher]( - pname, extension, version, target + attr_path, pname, extension, version, target ) successful_fetch = True break @@ -452,7 +453,7 @@ def _update_package(path, target): sri_hash = _hash_to_sri("sha256", new_sha256) # retrieve the old output hash for a more precise match - if old_hash := _get_attr_value(f"python3Packages.{pname}.src.outputHash"): + if old_hash := _get_attr_value(f"{attr_path}.src.outputHash"): # fetchers can specify a sha256, or a sri hash try: text = _replace_value("hash", sri_hash, text, old_hash) |