diff options
author | Frederik Rietdijk <fridh@fridh.nl> | 2016-12-03 13:04:52 +0100 |
---|---|---|
committer | Frederik Rietdijk <fridh@fridh.nl> | 2016-12-05 10:29:22 +0100 |
commit | 42bad32e130378428647d88d6809f23591b85606 (patch) | |
tree | 63222daf45af3bbf50dad32b4fc380794b55e852 | |
parent | c6a7d8e10f1e04ccef93d51459697c4b0a3a8f66 (diff) | |
download | nixlib-42bad32e130378428647d88d6809f23591b85606.tar nixlib-42bad32e130378428647d88d6809f23591b85606.tar.gz nixlib-42bad32e130378428647d88d6809f23591b85606.tar.bz2 nixlib-42bad32e130378428647d88d6809f23591b85606.tar.lz nixlib-42bad32e130378428647d88d6809f23591b85606.tar.xz nixlib-42bad32e130378428647d88d6809f23591b85606.tar.zst nixlib-42bad32e130378428647d88d6809f23591b85606.zip |
buildPythonPackage: refactor
8 files changed, 149 insertions, 111 deletions
diff --git a/doc/languages-frameworks/python.md b/doc/languages-frameworks/python.md index fc0100c5f2d5..e7a8b034a112 100644 --- a/doc/languages-frameworks/python.md +++ b/doc/languages-frameworks/python.md @@ -523,7 +523,7 @@ All parameters from `mkDerivation` function are still supported. * `postShellHook`: Hook to execute commands after `shellHook`. * `makeWrapperArgs`: A list of strings. Arguments to be passed to `makeWrapper`, which wraps generated binaries. By default, the arguments to `makeWrapper` set `PATH` and `PYTHONPATH` environment variables before calling the binary. Additional arguments here can allow a developer to set environment variables which will be available when the binary is run. For example, `makeWrapperArgs = ["--set FOO BAR" "--set BAZ QUX"]`. * `installFlags`: A list of strings. Arguments to be passed to `pip install`. To pass options to `python setup.py install`, use `--install-option`. E.g., `installFlags=["--install-option='--cpp_implementation'"]. -* `format`: Format of the source. Options are `setup` for when the source has a `setup.py` and `setuptools` is used to build a wheel, and `wheel` in case the source is already a binary wheel. The default value is `setup`. +* `format`: Format of the source. Valid options are `setuptools` (default), `flit`, `wheel`, and `other`. `setuptools` is for when the source has a `setup.py` and `setuptools` is used to build a wheel, `flit`, in case `flit` should be used to build a wheel, and `wheel` in case a wheel is provided. In case you need to provide your own `buildPhase` and `installPhase` you can use `other`. * `catchConflicts` If `true`, abort package build if a package name appears more than once in dependency tree. Default is `true`. * `checkInputs` Dependencies needed for running the `checkPhase`. These are added to `buildInputs` when `doCheck = true`. diff --git a/pkgs/development/interpreters/python/build-python-package-common.nix b/pkgs/development/interpreters/python/build-python-package-common.nix new file mode 100644 index 000000000000..80d40013f157 --- /dev/null +++ b/pkgs/development/interpreters/python/build-python-package-common.nix @@ -0,0 +1,27 @@ +# This function provides generic bits to install a Python wheel. + +{ python +, bootstrapped-pip +}: + +{ buildInputs ? [] +# Additional flags to pass to "pip install". +, installFlags ? [] +, ... } @ attrs: + +attrs // { + buildInputs = buildInputs ++ [ bootstrapped-pip ]; + + installPhase = attrs.installPhase or '' + runHook preInstall + + mkdir -p "$out/${python.sitePackages}" + export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" + + pushd dist + ${bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} + popd + + runHook postInstall + ''; +} \ No newline at end of file diff --git a/pkgs/development/interpreters/python/build-python-package-flit.nix b/pkgs/development/interpreters/python/build-python-package-flit.nix new file mode 100644 index 000000000000..8628c3df769e --- /dev/null +++ b/pkgs/development/interpreters/python/build-python-package-flit.nix @@ -0,0 +1,19 @@ +# This function provides specific bits for building a flit-based Python package. + +{ flit +}: + +{ ... } @ attrs: + +attrs // { + buildInputs = [ flit ]; + buildPhase = attrs.buildPhase or '' + runHook preBuild + flit wheel + runHook postBuild + ''; + + # Flit packages do not come with tests. + installCheckPhase = attrs.checkPhase or ":"; + doCheck = attrs.doCheck or false; +} \ No newline at end of file diff --git a/pkgs/development/interpreters/python/build-python-package-setuptools.nix b/pkgs/development/interpreters/python/build-python-package-setuptools.nix new file mode 100644 index 000000000000..f077533ecfe4 --- /dev/null +++ b/pkgs/development/interpreters/python/build-python-package-setuptools.nix @@ -0,0 +1,56 @@ +# This function provides specific bits for building a setuptools-based Python package. + +{ lib +, python +, bootstrapped-pip +}: + +{ +# passed to "python setup.py build_ext" +# https://github.com/pypa/pip/issues/881 + setupPyBuildFlags ? [] +# Execute before shell hook +, preShellHook ? "" +# Execute after shell hook +, postShellHook ? "" +, ... } @ attrs: + +let + # use setuptools shim (so that setuptools is imported before distutils) + # pip does the same thing: https://github.com/pypa/pip/pull/3265 + setuppy = ./run_setup.py; + +in attrs // { + # we copy nix_run_setup.py over so it's executed relative to the root of the source + # many project make that assumption + buildPhase = attrs.buildPhase or '' + runHook preBuild + cp ${setuppy} nix_run_setup.py + ${python.interpreter} nix_run_setup.py ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel + runHook postBuild + ''; + + installCheckPhase = attrs.checkPhase or '' + runHook preCheck + ${python.interpreter} nix_run_setup.py test + runHook postCheck + ''; + + # Python packages that are installed with setuptools + # are typically distributed with tests. + # With Python it's a common idiom to run the tests + # after the software has been installed. + doCheck = attrs.doCheck or true; + + shellHook = attrs.shellHook or '' + ${preShellHook} + if test -e setup.py; then + tmp_path=$(mktemp -d) + export PATH="$tmp_path/bin:$PATH" + export PYTHONPATH="$tmp_path/${python.sitePackages}:$PYTHONPATH" + mkdir -p $tmp_path/${python.sitePackages} + ${bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path + fi + ${postShellHook} + ''; +} \ No newline at end of file diff --git a/pkgs/development/interpreters/python/build-python-package-wheel.nix b/pkgs/development/interpreters/python/build-python-package-wheel.nix new file mode 100644 index 000000000000..7be0a4c304a3 --- /dev/null +++ b/pkgs/development/interpreters/python/build-python-package-wheel.nix @@ -0,0 +1,20 @@ +# This function provides specific bits for building a wheel-based Python package. + +{ +}: + +{ ... } @ attrs: + +attrs // { + unpackPhase = '' + mkdir dist + cp $src dist/"''${src#*-}" + ''; + + # Wheels are pre-compiled + buildPhase = attrs.buildPhase or ":"; + installCheckPhase = attrs.checkPhase or ":"; + + # Wheels don't have any checks to run + doCheck = attrs.doCheck or false; +} \ No newline at end of file diff --git a/pkgs/development/interpreters/python/build-python-package.nix b/pkgs/development/interpreters/python/build-python-package.nix index a92296cedbaa..e15405e2981d 100644 --- a/pkgs/development/interpreters/python/build-python-package.nix +++ b/pkgs/development/interpreters/python/build-python-package.nix @@ -7,120 +7,31 @@ , python , mkPythonDerivation , bootstrapped-pip +, flit }: -{ buildInputs ? [] - -# propagate build dependencies so in case we have A -> B -> C, -# C can import package A propagated by B -#, propagatedBuildInputs ? [] - -# passed to "python setup.py build_ext" -# https://github.com/pypa/pip/issues/881 -, setupPyBuildFlags ? [] - -# Execute before shell hook -, preShellHook ? "" - -# Execute after shell hook -, postShellHook ? "" - -# Additional flags to pass to "pip install". -, installFlags ? [] - -, format ? "setup" - +let + setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python bootstrapped-pip; }; + flit-specific = import ./build-python-package-flit.nix { inherit flit; }; + wheel-specific = import ./build-python-package-wheel.nix { }; + common = import ./build-python-package-common.nix { inherit python bootstrapped-pip; }; +in + +{ +# Several package formats are supported. +# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel. +# "wheel" : Install from a pre-compiled wheel. +# "flit" : Install a flit package. This builds a wheel. +# "other" : Provide your own buildPhase and installPhase. +format ? "setuptools" , ... } @ attrs: - - - let - # use setuptools shim (so that setuptools is imported before distutils) - # pip does the same thing: https://github.com/pypa/pip/pull/3265 - setuppy = ./run_setup.py; - formatspecific = - if format == "wheel" then - { - unpackPhase = '' - mkdir dist - cp $src dist/"''${src#*-}" - ''; - - # Wheels are pre-compiled - buildPhase = attrs.buildPhase or ":"; - installCheckPhase = attrs.checkPhase or ":"; - - # Wheels don't have any checks to run - doCheck = attrs.doCheck or false; - } - else if format == "setup" then - { - # we copy nix_run_setup.py over so it's executed relative to the root of the source - # many project make that assumption - buildPhase = attrs.buildPhase or '' - runHook preBuild - cp ${setuppy} nix_run_setup.py - ${python.interpreter} nix_run_setup.py ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel - runHook postBuild - ''; - - installCheckPhase = attrs.checkPhase or '' - runHook preCheck - ${python.interpreter} nix_run_setup.py test - runHook postCheck - ''; - - # Python packages that are installed with setuptools - # are typically distributed with tests. - # With Python it's a common idiom to run the tests - # after the software has been installed. - doCheck = attrs.doCheck or true; - } - else - throw "Unsupported format ${format}"; - -in mkPythonDerivation ( attrs // { - - # To build and install a wheel we need pip - buildInputs = buildInputs ++ [ bootstrapped-pip ]; - -#inherit propagatedBuildInputs; - - configurePhase = attrs.configurePhase or '' - runHook preConfigure - - # patch python interpreter to write null timestamps when compiling python files - # this way python doesn't try to update them when we freeze timestamps in nix store - export DETERMINISTIC_BUILD=1 - - runHook postConfigure - ''; - - installPhase = attrs.installPhase or '' - runHook preInstall - - mkdir -p "$out/${python.sitePackages}" - export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" - - pushd dist - ${bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} - popd - - runHook postInstall - ''; - - shellHook = attrs.shellHook or '' - ${preShellHook} - if test -e setup.py; then - tmp_path=$(mktemp -d) - export PATH="$tmp_path/bin:$PATH" - export PYTHONPATH="$tmp_path/${python.sitePackages}:$PYTHONPATH" - mkdir -p $tmp_path/${python.sitePackages} - ${bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path - fi - ${postShellHook} - ''; + if format == "setuptools" then common (setuptools-specific attrs) + else if format == "flit" then common (flit-specific attrs) + else if format == "wheel" then common (wheel-specific attrs) + else if format == "other" then {} + else throw "Unsupported format ${format}"; -} // formatspecific) +in mkPythonDerivation ( attrs // formatspecific ) \ No newline at end of file diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 0f798c63e8fa..c8fedaf75fc9 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -57,6 +57,10 @@ python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled"] // { inherit pythonPath; + # patch python interpreter to write null timestamps when compiling python files + # this way python doesn't try to update them when we freeze timestamps in nix store + DETERMINISTIC_BUILD=1; + buildInputs = [ wrapPython ] ++ buildInputs ++ pythonPath ++ [ (ensureNewerSourcesHook { year = "1980"; }) ] ++ (lib.optional (lib.hasSuffix "zip" attrs.src.name or "") unzip) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 3575018870f3..122042072755 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -30,6 +30,7 @@ let buildPythonPackage = makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix { inherit mkPythonDerivation; inherit bootstrapped-pip; + flit = self.flit; }); buildPythonApplication = args: buildPythonPackage ({namePrefix="";} // args ); |