Merge pull request #12826 from NixOS/python-wip

pythonPackages: new functions to build numpy and scipy

6 files changed, 191 insertions, 108 deletions

diff --git a/pkgs/development/libraries/science/math/openblas/0.2.14.nix b/pkgs/development/libraries/science/math/openblas/0.2.14.nix
new file mode 100644
index 000000000000..2fac8a4db08b
--- /dev/null
+++ b/pkgs/development/libraries/science/math/openblas/0.2.14.nix
@@ -0,0 +1,65 @@
+{ stdenv, fetchurl, gfortran, perl, which, config, coreutils
+# Most packages depending on openblas expect integer width to match pointer width,
+# but some expect to use 32-bit integers always (for compatibility with reference BLAS).
+, blas64 ? null
+}:
+
+with stdenv.lib;
+
+let blas64_ = blas64; in
+
+let local = config.openblas.preferLocalBuild or false;
+    binary =
+      { i686-linux = "32";
+        x86_64-linux = "64";
+        x86_64-darwin = "64";
+      }."${stdenv.system}" or (throw "unsupported system: ${stdenv.system}");
+    genericFlags =
+      [ "DYNAMIC_ARCH=1"
+        "NUM_THREADS=64"
+      ];
+    localFlags = config.openblas.flags or
+      optionals (hasAttr "target" config.openblas) [ "TARGET=${config.openblas.target}" ];
+    blas64 = if blas64_ != null then blas64_ else hasPrefix "x86_64" stdenv.system;
+
+    version = "0.2.14";
+in
+stdenv.mkDerivation {
+  name = "openblas-${version}";
+  src = fetchurl {
+    url = "https://github.com/xianyi/OpenBLAS/archive/v${version}.tar.gz";
+    sha256 = "2411c4f56f477b42dff54db2b7ffc0b7cf53bb9778d54982595c64cc69c40fc1";
+    name = "openblas-${version}.tar.gz";
+  };
+
+  inherit blas64;
+
+  nativeBuildInputs = optionals stdenv.isDarwin [coreutils] ++ [gfortran perl which];
+
+  makeFlags =
+    (if local then localFlags else genericFlags)
+    ++
+    optionals stdenv.isDarwin ["MACOSX_DEPLOYMENT_TARGET=10.9"]
+    ++
+    [
+      "FC=gfortran"
+      # Note that clang is available through the stdenv on OSX and
+      # thus is not an explicit dependency.
+      "CC=${if stdenv.isDarwin then "clang" else "gcc"}"
+      ''PREFIX="''$(out)"''
+      "BINARY=${binary}"
+      "USE_OPENMP=${if stdenv.isDarwin then "0" else "1"}"
+      "INTERFACE64=${if blas64 then "1" else "0"}"
+    ];
+
+  doCheck = true;
+  checkTarget = "tests";
+
+  meta = with stdenv.lib; {
+    description = "Basic Linear Algebra Subprograms";
+    license = licenses.bsd3;
+    homepage = "https://github.com/xianyi/OpenBLAS";
+    platforms = platforms.unix;
+    maintainers = with maintainers; [ ttuegel ];
+  };
+}
diff --git a/pkgs/development/python-modules/numpy-scipy-support.nix b/pkgs/development/python-modules/numpy-scipy-support.nix
deleted file mode 100644
index 422de794e31b..000000000000
--- a/pkgs/development/python-modules/numpy-scipy-support.nix
+++ /dev/null
@@ -1,35 +0,0 @@
-{
-  # Python package expression
-  python,
-  # Name of package (e.g. numpy or scipy)
-  pkgName,
-  # OpenBLAS math library
-  openblas
-}:
-
-{
-  # Re-export openblas here so that it can be sure that the same one will be used
-  # in the propagatedBuildInputs.
-  inherit openblas;
-
-  # First "install" the package, then import what was installed, and call the
-  # .test() function, which will run the test suite.
-  checkPhase = ''
-    runHook preCheck
-    pushd dist
-    ${python.interpreter} -c 'import ${pkgName}; ${pkgName}.test("fast", verbose=10)'
-    popd
-    runHook postCheck
-  '';
-
-  # Creates a site.cfg telling the setup script where to find depended-on
-  # math libraries.
-  preBuild = ''
-    echo "Creating site.cfg file..."
-    cat << EOF > site.cfg
-    [openblas]
-    include_dirs = ${openblas}/include
-    library_dirs = ${openblas}/lib
-    EOF
-  '';
-}
diff --git a/pkgs/development/python-modules/numpy.nix b/pkgs/development/python-modules/numpy.nix
new file mode 100644
index 000000000000..141c8b14fa6d
--- /dev/null
+++ b/pkgs/development/python-modules/numpy.nix
@@ -0,0 +1,51 @@
+{lib, python, buildPythonPackage, isPyPy, gfortran, nose, blas}:
+
+args:
+
+let
+  inherit (args) version;
+in buildPythonPackage (args // rec {
+
+  name = "numpy-${version}";
+
+  disabled = isPyPy;
+  buildInputs = args.buildInputs or [ gfortran nose ];
+  propagatedBuildInputs = args.propagatedBuildInputs or [ passthru.blas ];
+
+  preConfigure = ''
+    sed -i 's/-faltivec//' numpy/distutils/system_info.py
+  '';
+
+  preBuild = ''
+    echo "Creating site.cfg file..."
+    cat << EOF > site.cfg
+    [openblas]
+    include_dirs = ${passthru.blas}/include
+    library_dirs = ${passthru.blas}/lib
+    EOF
+  '';
+
+  checkPhase = ''
+    runHook preCheck
+    pushd dist
+    ${python.interpreter} -c 'import numpy; numpy.test("fast", verbose=10)'
+    popd
+    runHook postCheck
+  '';
+
+  passthru = {
+    blas = blas;
+  };
+
+  # The large file support test is disabled because it takes forever
+  # and can cause the machine to run out of disk space when run.
+  prePatch = ''
+    sed -i 's/test_large_file_support/donttest/' numpy/lib/tests/test_format.py
+  '';
+
+  meta = {
+    description = "Scientific tools for Python";
+    homepage = "http://numpy.scipy.org/";
+    maintainers = with lib.maintainers; [ fridh ];
+  } // (args.meta or {});
+})
diff --git a/pkgs/development/python-modules/scipy.nix b/pkgs/development/python-modules/scipy.nix
new file mode 100644
index 000000000000..ae312cd32d59
--- /dev/null
+++ b/pkgs/development/python-modules/scipy.nix
@@ -0,0 +1,47 @@
+{lib, python, buildPythonPackage, isPyPy, gfortran, nose}:
+
+args:
+
+let
+  inherit (args) version;
+  inherit (args) numpy;
+in buildPythonPackage (args // rec {
+
+  name = "scipy-${version}";
+
+  buildInputs = (args.buildInputs or [ gfortran nose ]);
+  propagatedBuildInputs = (args.propagatedBuildInputs or [ passthru.blas numpy]);
+
+  preConfigure = ''
+    sed -i '0,/from numpy.distutils.core/s//import setuptools;from numpy.distutils.core/' setup.py
+  '';
+
+  preBuild = ''
+    echo "Creating site.cfg file..."
+    cat << EOF > site.cfg
+    [openblas]
+    include_dirs = ${passthru.blas}/include
+    library_dirs = ${passthru.blas}/lib
+    EOF
+  '';
+
+  checkPhase = ''
+    runHook preCheck
+    pushd dist
+    ${python.interpreter} -c 'import scipy; scipy.test("fast", verbose=10)'
+    popd
+    runHook postCheck
+  '';
+
+  passthru = {
+    blas = numpy.blas;
+  };
+
+  setupPyBuildFlags = [ "--fcompiler='gnu95'" ];
+
+  meta = {
+    description = "SciPy (pronounced 'Sigh Pie') is open-source software for mathematics, science, and engineering. ";
+    homepage = http://www.scipy.org/;
+    maintainers = with lib.maintainers; [ fridh ];
+  } // (args.meta or {});
+})
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index e103d1e5b536..2eec8c8d3dba 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -15029,10 +15029,12 @@ let
   liblbfgs = callPackage ../development/libraries/science/math/liblbfgs { };
 
   openblas = callPackage ../development/libraries/science/math/openblas { };
+  openblas_2_14 = callPackage ../development/libraries/science/math/openblas/0.2.14.nix { };
 
   # A version of OpenBLAS using 32-bit integers on all platforms for compatibility with
   # standard BLAS and LAPACK.
   openblasCompat = openblas.override { blas64 = false; };
+  openblasCompat_2_14 = openblas_2_14.override { blas64 = false; };
 
   openlibm = callPackage ../development/libraries/science/math/openlibm {};
 
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 18671f3b4c73..55e36cbe71e9 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -12505,48 +12505,20 @@ in modules // {
     };
   };
 
-  numpy = let
-    support = import ../development/python-modules/numpy-scipy-support.nix {
-      inherit python;
-      openblas = pkgs.openblasCompat;
-      pkgName = "numpy";
-    };
-  in buildPythonPackage ( rec {
-    name = "numpy-${version}";
-    version = "1.10.4";
+  buildNumpyPackage = callPackage ../development/python-modules/numpy.nix {
+    gfortran = pkgs.gfortran;
+    blas = pkgs.openblasCompat_2_14;
+  };
+
+  numpy = self.numpy_1_10;
 
+  numpy_1_10 = self.buildNumpyPackage rec {
+    version = "1.10.4";
     src = pkgs.fetchurl {
-      url = "https://pypi.python.org/packages/source/n/numpy/${name}.tar.gz";
+      url = "https://pypi.python.org/packages/source/n/numpy/numpy-${version}.tar.gz";
       sha256 = "7356e98fbcc529e8d540666f5a919912752e569150e9a4f8d869c686f14c720b";
     };
-
-    disabled = isPyPy;  # WIP
-
-    preConfigure = ''
-      sed -i 's/-faltivec//' numpy/distutils/system_info.py
-    '';
-
-    inherit (support) preBuild checkPhase;
-
-    buildInputs = [ pkgs.gfortran self.nose ];
-    propagatedBuildInputs = [ support.openblas ];
-
-    # Disable failing test_f2py test.
-    # f2py couldn't be found by test,
-    # even though it was used successfully to build numpy
-
-    # The large file support test is disabled because it takes forever
-    # and can cause the machine to run out of disk space when run.
-    prePatch = ''
-      sed -i 's/test_f2py/donttest/' numpy/tests/test_scripts.py
-      sed -i 's/test_large_file_support/donttest/' numpy/lib/tests/test_format.py
-    '';
-
-    meta = {
-      description = "Scientific tools for Python";
-      homepage = "http://numpy.scipy.org/";
-    };
-  });
+  };
 
   numpydoc = buildPythonPackage rec {
     name = "numpydoc-${version}";
@@ -18488,47 +18460,28 @@ in modules // {
     };
   };
 
+  buildScipyPackage = callPackage ../development/python-modules/scipy.nix {
+    gfortran = pkgs.gfortran;
+  };
 
-  scipy = let
-    support = import ../development/python-modules/numpy-scipy-support.nix {
-      inherit python;
-      openblas = pkgs.openblasCompat;
-      pkgName = "scipy";
-    };
-  in buildPythonPackage rec {
-    name = "scipy-${version}";
-    version = "0.16.1";
+  scipy = self.scipy_0_17;
 
+  scipy_0_16 = self.buildScipyPackage rec {
+    version = "0.16.1";
     src = pkgs.fetchurl {
-      url = "http://pypi.python.org/packages/source/s/scipy/${name}.tar.gz";
+      url = "https://pypi.python.org/packages/source/s/scipy/scipy-${version}.tar.gz";
       sha256 = "ecd1efbb1c038accb0516151d1e6679809c6010288765eb5da6051550bf52260";
     };
+    numpy = self.numpy_1_10;
+  };
 
-    buildInputs = [ pkgs.gfortran self.nose ];
-    propagatedBuildInputs = [ self.numpy ];
-
-    preConfigure = ''
-      sed -i '0,/from numpy.distutils.core/s//import setuptools;from numpy.distutils.core/' setup.py
-    '';
-
-    # First test: RuntimeWarning: Mean of empty slice.
-    # Second: SyntaxError: invalid syntax. Due to wrapper?
-    # Third: test checks permissions
-    prePatch = ''
-      substituteInPlace scipy/stats/tests/test_stats.py --replace "test_chisquare_masked_arrays" "remove_this_one"
-      rm scipy/linalg/tests/test_lapack.py
-      substituteInPlace scipy/weave/tests/test_catalog.py --replace "test_user" "remove_this_one"
-    '';
-
-    inherit (support) preBuild checkPhase;
-
-    patches = [../development/python-modules/scipy-0.16.1-decorator-fix.patch];
-    setupPyBuildFlags = [ "--fcompiler='gnu95'" ];
-
-    meta = {
-      description = "SciPy (pronounced 'Sigh Pie') is open-source software for mathematics, science, and engineering. ";
-      homepage = http://www.scipy.org/;
+  scipy_0_17 = self.buildScipyPackage rec {
+    version = "0.17.0";
+    src = pkgs.fetchurl {
+      url = "https://pypi.python.org/packages/source/s/scipy/scipy-${version}.tar.gz";
+      sha256 = "f600b755fb69437d0f70361f9e560ab4d304b1b66987ed5a28bdd9dd7793e089";
     };
+    numpy = self.numpy_1_10;
   };
 
   scikitimage = buildPythonPackage rec {
@@ -18563,7 +18516,7 @@ in modules // {
     };
 
     buildInputs = with self; [ nose pillow pkgs.gfortran pkgs.glibcLocales ];
-    propagatedBuildInputs = with self; [ numpy scipy pkgs.openblas ];
+    propagatedBuildInputs = with self; [ numpy scipy numpy.blas ];
 
     LC_ALL="en_US.UTF-8";