diff options
author | Frederik Rietdijk <fridh@fridh.nl> | 2016-08-31 10:13:15 +0200 |
---|---|---|
committer | Frederik Rietdijk <fridh@fridh.nl> | 2016-09-01 16:16:31 +0200 |
commit | b080748d42d18a3a51e4afacc962d9528ae4200d (patch) | |
tree | 93aad640de499a1473afadf22207468bc2f62b25 /pkgs/development/interpreters | |
parent | b17cd148ba7a9eaadd689c49e8b72c80151dcecb (diff) | |
download | nixlib-b080748d42d18a3a51e4afacc962d9528ae4200d.tar nixlib-b080748d42d18a3a51e4afacc962d9528ae4200d.tar.gz nixlib-b080748d42d18a3a51e4afacc962d9528ae4200d.tar.bz2 nixlib-b080748d42d18a3a51e4afacc962d9528ae4200d.tar.lz nixlib-b080748d42d18a3a51e4afacc962d9528ae4200d.tar.xz nixlib-b080748d42d18a3a51e4afacc962d9528ae4200d.tar.zst nixlib-b080748d42d18a3a51e4afacc962d9528ae4200d.zip |
Python: move python-modules/generic to interpreter folder
Diffstat (limited to 'pkgs/development/interpreters')
-rw-r--r-- | pkgs/development/interpreters/python/buildpythonpackage.nix | 174 | ||||
-rw-r--r-- | pkgs/development/interpreters/python/catch_conflicts.py | 30 | ||||
-rw-r--r-- | pkgs/development/interpreters/python/run_setup.py | 8 | ||||
-rw-r--r-- | pkgs/development/interpreters/python/wrap.sh | 118 |
4 files changed, 330 insertions, 0 deletions
diff --git a/pkgs/development/interpreters/python/buildpythonpackage.nix b/pkgs/development/interpreters/python/buildpythonpackage.nix new file mode 100644 index 000000000000..38d74e082445 --- /dev/null +++ b/pkgs/development/interpreters/python/buildpythonpackage.nix @@ -0,0 +1,174 @@ +/* This function provides a generic Python package builder. It is + intended to work with packages that use `distutils/setuptools' + (http://pypi.python.org/pypi/setuptools/), which represents a large + number of Python packages nowadays. */ + +{ python, setuptools, unzip, wrapPython, lib, bootstrapped-pip +, ensureNewerSourcesHook }: + +{ name + +# by default prefix `name` e.g. "python3.3-${name}" +, namePrefix ? python.libPrefix + "-" + +, buildInputs ? [] + +# propagate build dependencies so in case we have A -> B -> C, +# C can import package A propagated by B +, propagatedBuildInputs ? [] + +# passed to "python setup.py build_ext" +# https://github.com/pypa/pip/issues/881 +, setupPyBuildFlags ? [] + +# DEPRECATED: use propagatedBuildInputs +, pythonPath ? [] + +# used to disable derivation, useful for specific python versions +, disabled ? false + +, meta ? {} + +# Execute before shell hook +, preShellHook ? "" + +# Execute after shell hook +, postShellHook ? "" + +# Additional arguments to pass to the makeWrapper function, which wraps +# generated binaries. +, makeWrapperArgs ? [] + +# Additional flags to pass to "pip install". +, installFlags ? [] + +# Raise an error if two packages are installed with the same name +, catchConflicts ? true + +, format ? "setup" + +, ... } @ attrs: + + +# Keep extra attributes from `attrs`, e.g., `patchPhase', etc. +if disabled +then throw "${name} not supported for interpreter ${python.executable}" +else + +let + # use setuptools shim (so that setuptools is imported before distutils) + # pip does the same thing: https://github.com/pypa/pip/pull/3265 + setuppy = ./run_setup.py; + + formatspecific = + if format == "wheel" then + { + unpackPhase = '' + mkdir dist + cp $src dist/"''${src#*-}" + ''; + + # Wheels are pre-compiled + buildPhase = attrs.buildPhase or ":"; + installCheckPhase = attrs.checkPhase or ":"; + + # Wheels don't have any checks to run + doInstallCheck = attrs.doCheck or false; + } + else if format == "setup" then + { + # propagate python/setuptools to active setup-hook in nix-shell + propagatedBuildInputs = + propagatedBuildInputs ++ [ python setuptools ]; + + # we copy nix_run_setup.py over so it's executed relative to the root of the source + # many project make that assumption + buildPhase = attrs.buildPhase or '' + runHook preBuild + cp ${setuppy} nix_run_setup.py + ${python.interpreter} nix_run_setup.py ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel + runHook postBuild + ''; + + installCheckPhase = attrs.checkPhase or '' + runHook preCheck + ${python.interpreter} nix_run_setup.py test + runHook postCheck + ''; + + # Python packages that are installed with setuptools + # are typically distributed with tests. + # With Python it's a common idiom to run the tests + # after the software has been installed. + + # For backwards compatibility, let's use an alias + doInstallCheck = attrs.doCheck or true; + } + else + throw "Unsupported format ${format}"; +in +python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled" "doCheck"] // { + name = namePrefix + name; + + buildInputs = [ wrapPython bootstrapped-pip ] ++ buildInputs ++ pythonPath + ++ [ (ensureNewerSourcesHook { year = "1980"; }) ] + ++ (lib.optional (lib.hasSuffix "zip" attrs.src.name or "") unzip); + + pythonPath = pythonPath; + + configurePhase = attrs.configurePhase or '' + runHook preConfigure + + # patch python interpreter to write null timestamps when compiling python files + # this way python doesn't try to update them when we freeze timestamps in nix store + export DETERMINISTIC_BUILD=1 + + runHook postConfigure + ''; + + # Python packages don't have a checkPhase, only an installCheckPhase + doCheck = false; + + installPhase = attrs.installPhase or '' + runHook preInstall + + mkdir -p "$out/${python.sitePackages}" + export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" + + pushd dist + ${bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} + popd + + runHook postInstall + ''; + + postFixup = attrs.postFixup or '' + wrapPythonPrograms + '' + lib.optionalString catchConflicts '' + # check if we have two packages with the same name in closure and fail + # this shouldn't happen, something went wrong with dependencies specs + ${python.interpreter} ${./catch_conflicts.py} + ''; + + shellHook = attrs.shellHook or '' + ${preShellHook} + if test -e setup.py; then + tmp_path=$(mktemp -d) + export PATH="$tmp_path/bin:$PATH" + export PYTHONPATH="$tmp_path/${python.sitePackages}:$PYTHONPATH" + mkdir -p $tmp_path/${python.sitePackages} + ${bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path + fi + ${postShellHook} + ''; + + meta = with lib.maintainers; { + # default to python's platforms + platforms = python.meta.platforms; + } // meta // { + # add extra maintainer(s) to every package + maintainers = (meta.maintainers or []) ++ [ chaoflow domenkozar ]; + # a marker for release utilities to discover python packages + isBuildPythonPackage = python.meta.platforms; + }; +} // formatspecific) diff --git a/pkgs/development/interpreters/python/catch_conflicts.py b/pkgs/development/interpreters/python/catch_conflicts.py new file mode 100644 index 000000000000..bb82900c65a9 --- /dev/null +++ b/pkgs/development/interpreters/python/catch_conflicts.py @@ -0,0 +1,30 @@ +import pkg_resources +import collections +import sys + +do_abort = False +packages = collections.defaultdict(list) + +for f in sys.path: + for req in pkg_resources.find_distributions(f): + if req not in packages[req.project_name]: + # some exceptions inside buildPythonPackage + if req.project_name in ['setuptools', 'pip', 'wheel']: + continue + packages[req.project_name].append(req) + + +for name, duplicates in packages.items(): + if len(duplicates) > 1: + do_abort = True + print("Found duplicated packages in closure for dependency '{}': ".format(name)) + for dup in duplicates: + print(" " + repr(dup)) + +if do_abort: + print("") + print( + 'Package duplicates found in closure, see above. Usually this ' + 'happens if two packages depend on different version ' + 'of the same dependency.') + sys.exit(1) diff --git a/pkgs/development/interpreters/python/run_setup.py b/pkgs/development/interpreters/python/run_setup.py new file mode 100644 index 000000000000..e3a530eb0cb6 --- /dev/null +++ b/pkgs/development/interpreters/python/run_setup.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +import setuptools +import tokenize + +__file__='setup.py'; + +exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec')) diff --git a/pkgs/development/interpreters/python/wrap.sh b/pkgs/development/interpreters/python/wrap.sh new file mode 100644 index 000000000000..f4b63b826403 --- /dev/null +++ b/pkgs/development/interpreters/python/wrap.sh @@ -0,0 +1,118 @@ +# Wrapper around wrapPythonProgramsIn, below. The $pythonPath +# variable is passed in from the buildPythonPackage function. +wrapPythonPrograms() { + wrapPythonProgramsIn $out "$out $pythonPath" +} + +# Builds environment variables like PYTHONPATH and PATH walking through closure +# of dependencies. +buildPythonPath() { + local pythonPath="$1" + local path + + # Create an empty table of python paths (see doc on _addToPythonPath + # for how this is used). Build up the program_PATH and program_PYTHONPATH + # variables. + declare -A pythonPathsSeen=() + program_PYTHONPATH= + program_PATH= + pythonPathsSeen["@python@"]=1 + addToSearchPath program_PATH @python@/bin + for path in $pythonPath; do + _addToPythonPath $path + done +} + +# Patches a Python script so that it has correct libraries path and executable +# name. +patchPythonScript() { + local f="$1" + + # The magicalSedExpression will invoke a "$(basename "$f")", so + # if you change $f to something else, be sure to also change it + # in pkgs/top-level/python-packages.nix! + # It also uses $program_PYTHONPATH. + sed -i "$f" -re '@magicalSedExpression@' +} + +# Transforms any binaries generated by the setup.py script, replacing them +# with an executable shell script which will set some environment variables +# and then call into the original binary (which has been given a .wrapped +# suffix). +wrapPythonProgramsIn() { + local dir="$1" + local pythonPath="$2" + local f + + buildPythonPath "$pythonPath" + + # Find all regular files in the output directory that are executable. + for f in $(find "$dir" -type f -perm -0100); do + # Rewrite "#! .../env python" to "#! /nix/store/.../python". + # Strip suffix, like "3" or "2.7m" -- we don't have any choice on which + # Python to use besides one with this hook anyway. + if head -n1 "$f" | grep -q '#!.*/env.*\(python\|pypy\)'; then + sed -i "$f" -e "1 s^.*/env[ ]*\(python\|pypy\)[^ ]*^#! @executable@^" + fi + + # catch /python and /.python-wrapped + if head -n1 "$f" | grep -q '/\.\?\(python\|pypy\)'; then + # dont wrap EGG-INFO scripts since they are called from python + if echo "$f" | grep -qv EGG-INFO/scripts; then + echo "wrapping \`$f'..." + patchPythonScript "$f" + # wrapProgram creates the executable shell script described + # above. The script will set PYTHONPATH and PATH variables.! + # (see pkgs/build-support/setup-hooks/make-wrapper.sh) + local -a wrap_args=("$f" + --prefix PATH ':' "$program_PATH") + + # Add any additional arguments provided by makeWrapperArgs + # argument to buildPythonPackage. + local -a user_args="($makeWrapperArgs)" + local -a wrapProgramArgs=("${wrap_args[@]}" "${user_args[@]}") + wrapProgram "${wrapProgramArgs[@]}" + fi + fi + done +} + +# Adds the lib and bin directories to the PYTHONPATH and PATH variables, +# respectively. Recurses on any paths declared in +# `propagated-native-build-inputs`, while avoiding duplicating paths by +# flagging the directories it has visited in `pythonPathsSeen`. +_addToPythonPath() { + local dir="$1" + # Stop if we've already visited here. + if [ -n "${pythonPathsSeen[$dir]}" ]; then return; fi + pythonPathsSeen[$dir]=1 + # addToSearchPath is defined in stdenv/generic/setup.sh. It will have + # the effect of calling `export program_X=$dir/...:$program_X`. + addToSearchPath program_PYTHONPATH $dir/lib/@libPrefix@/site-packages + addToSearchPath program_PATH $dir/bin + + # Inspect the propagated inputs (if they exist) and recur on them. + local prop="$dir/nix-support/propagated-native-build-inputs" + if [ -e $prop ]; then + local new_path + for new_path in $(cat $prop); do + _addToPythonPath $new_path + done + fi +} + +createBuildInputsPth() { + local category="$1" + local inputs="$2" + if [ foo"$inputs" != foo ]; then + for x in $inputs; do + if $(echo -n $x |grep -q python-recursive-pth-loader); then + continue + fi + if test -d "$x"/lib/@libPrefix@/site-packages; then + echo $x/lib/@libPrefix@/site-packages \ + >> "$out"/lib/@libPrefix@/site-packages/${name}-nix-python-$category.pth + fi + done + fi +} |