From 170dd558dffd2bb134e3001a93631fc71fe13f3c Mon Sep 17 00:00:00 2001 From: Nikolay Amiantov Date: Sat, 13 Jul 2019 22:37:01 +0300 Subject: python.pkgs.tensorflow: cleanup binary build --- pkgs/development/python-modules/tensorflow/bin.nix | 25 +++++++------ .../python-modules/tensorflow/binary-hashes.nix | 43 ++++++++++++++++++++++ .../python-modules/tensorflow/prefetcher.sh | 5 ++- .../python-modules/tensorflow/tf1.14.0-hashes.nix | 42 --------------------- 4 files changed, 61 insertions(+), 54 deletions(-) create mode 100644 pkgs/development/python-modules/tensorflow/binary-hashes.nix mode change 100644 => 100755 pkgs/development/python-modules/tensorflow/prefetcher.sh delete mode 100644 pkgs/development/python-modules/tensorflow/tf1.14.0-hashes.nix diff --git a/pkgs/development/python-modules/tensorflow/bin.nix b/pkgs/development/python-modules/tensorflow/bin.nix index 817ba09dc9d2..a562ba8f2fad 100644 --- a/pkgs/development/python-modules/tensorflow/bin.nix +++ b/pkgs/development/python-modules/tensorflow/bin.nix @@ -35,15 +35,19 @@ assert cudaSupport -> cudatoolkit != null && cudnn != null && nvidia_x11 != null; + +# unsupported combination +assert ! (stdenv.isDarwin && cudaSupport); + let - cudatoolkit_joined = symlinkJoin { - name = "unsplit_cudatoolkit"; - paths = [ cudatoolkit.out - cudatoolkit.lib ];}; + packages = import ./binary-hashes.nix; + + variant = if cudaSupport then "-gpu" else ""; + pname = "tensorflow${variant}"; in buildPythonPackage rec { - pname = "tensorflow"; - version = "1.14.0"; + inherit pname; + inherit (packages) version; format = "wheel"; src = let @@ -52,8 +56,7 @@ in buildPythonPackage rec { platform = if stdenv.isDarwin then "mac" else "linux"; unit = if cudaSupport then "gpu" else "cpu"; key = "${platform}_py_${pyver}_${unit}"; - dls = import (./. + "/tf${version}-hashes.nix"); - in fetchurl dls.${key}; + in fetchurl packages.${key}; propagatedBuildInputs = [ protobuf @@ -86,9 +89,9 @@ in buildPythonPackage rec { # patchelf --shrink-rpath will remove the cuda libraries. postFixup = let rpath = stdenv.lib.makeLibraryPath - ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit_joined cudnn nvidia_x11 ]); + ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]); in - lib.optionalString (stdenv.isLinux) '' + lib.optionalString stdenv.isLinux '' rrPath="$out/${python.sitePackages}/tensorflow/:$out/${python.sitePackages}/tensorflow/contrib/tensor_forest/:${rpath}" internalLibPath="$out/${python.sitePackages}/tensorflow/python/_pywrap_tensorflow_internal.so" find $out \( -name '*.so' -or -name '*.so.*' \) -exec patchelf --set-rpath "$rrPath" {} \; @@ -100,7 +103,7 @@ in buildPythonPackage rec { homepage = http://tensorflow.org; license = licenses.asl20; maintainers = with maintainers; [ jyp abbradar ]; - platforms = with platforms; linux ++ lib.optionals (!cudaSupport) darwin; + platforms = [ "x86_64-linux" "x86_64-darwin" ]; # Python 2.7 build uses different string encoding. # See https://github.com/NixOS/nixpkgs/pull/37044#issuecomment-373452253 broken = stdenv.isDarwin && !isPy3k; diff --git a/pkgs/development/python-modules/tensorflow/binary-hashes.nix b/pkgs/development/python-modules/tensorflow/binary-hashes.nix new file mode 100644 index 000000000000..fa4809dc3f19 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/binary-hashes.nix @@ -0,0 +1,43 @@ +{ +version = "1.14.0"; +linux_py_27_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp27-none-linux_x86_64.whl"; + sha256 = "0yywdrfk97dh1bxhibspg0raz70fx9lcczj6xlimqy4xb60clx7k"; +}; +linux_py_35_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp35-cp35m-linux_x86_64.whl"; + sha256 = "1xvyb6xcrjhlwvrmrhn5vs9xy7g98smqmpv4i3hhpry4qyasphhj"; +}; +linux_py_36_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl"; + sha256 = "1psd9vyxz9f39dwj77nvrg373sxv3p5vdp9fnz81dpsm0b0mwl44"; +}; +linux_py_37_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp37-cp37m-linux_x86_64.whl"; + sha256 = "0bg2sb1n2ag27r7ww695kg5hb0mjrw4kc5893krmixx2j71860c5"; +}; +linux_py_27_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp27-none-linux_x86_64.whl"; + sha256 = "0y1x91gayg6pjddgl8ndcm63wfzhyv4s5khgl7ffzsgni1ivaqw5"; +}; +linux_py_35_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp35-cp35m-linux_x86_64.whl"; + sha256 = "03piggpbz1jx8m2b95spq3jrdff4w6xx63ji07am7hyw2nsgx3mx"; +}; +linux_py_36_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp36-cp36m-linux_x86_64.whl"; + sha256 = "0ypkp8cfhharsyyikb1qgf44cfm6284km9xswzvzymjzz75vg3gd"; +}; +linux_py_37_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp37-cp37m-linux_x86_64.whl"; + sha256 = "0virp8nn2ysx4855hq29kas6fm6b3dsiybwzdxy9nnb9n2d8qlm2"; +}; +mac_py_2_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py2-none-any.whl"; + sha256 = "14f86k3pgq7z6i4s4im55zpp38f0drnm7xlclavsgcc0nxnj3z26"; +}; +mac_py_3_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py3-none-any.whl"; + sha256 = "0f3swpcjfgqhj6h5wnx8snc0xjkx4hnkqx83fmlrwpncs8c131d3"; +}; +} diff --git a/pkgs/development/python-modules/tensorflow/prefetcher.sh b/pkgs/development/python-modules/tensorflow/prefetcher.sh old mode 100644 new mode 100755 index 3059340d127d..d590fb0f1732 --- a/pkgs/development/python-modules/tensorflow/prefetcher.sh +++ b/pkgs/development/python-modules/tensorflow/prefetcher.sh @@ -1,7 +1,10 @@ +#!/usr/bin/env bash + version=1.14.0 -hashfile=tf${version}-hashes.nix +hashfile=binary-hashes.nix rm -f $hashfile echo "{" >> $hashfile +echo "version = \"$version\";" >> $hashfile for sys in "linux" "mac"; do for tfpref in "cpu/tensorflow" "gpu/tensorflow_gpu"; do for pykind in "py2-none-any" "py3-none-any" "cp27-none-linux_x86_64" "cp35-cp35m-linux_x86_64" "cp36-cp36m-linux_x86_64" "cp37-cp37m-linux_x86_64"; do diff --git a/pkgs/development/python-modules/tensorflow/tf1.14.0-hashes.nix b/pkgs/development/python-modules/tensorflow/tf1.14.0-hashes.nix deleted file mode 100644 index f6b367aea383..000000000000 --- a/pkgs/development/python-modules/tensorflow/tf1.14.0-hashes.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ -linux_py_27_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp27-none-linux_x86_64.whl"; - sha256 = "0yywdrfk97dh1bxhibspg0raz70fx9lcczj6xlimqy4xb60clx7k"; -}; -linux_py_35_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp35-cp35m-linux_x86_64.whl"; - sha256 = "1xvyb6xcrjhlwvrmrhn5vs9xy7g98smqmpv4i3hhpry4qyasphhj"; -}; -linux_py_36_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl"; - sha256 = "1psd9vyxz9f39dwj77nvrg373sxv3p5vdp9fnz81dpsm0b0mwl44"; -}; -linux_py_37_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp37-cp37m-linux_x86_64.whl"; - sha256 = "0bg2sb1n2ag27r7ww695kg5hb0mjrw4kc5893krmixx2j71860c5"; -}; -linux_py_27_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp27-none-linux_x86_64.whl"; - sha256 = "0y1x91gayg6pjddgl8ndcm63wfzhyv4s5khgl7ffzsgni1ivaqw5"; -}; -linux_py_35_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp35-cp35m-linux_x86_64.whl"; - sha256 = "03piggpbz1jx8m2b95spq3jrdff4w6xx63ji07am7hyw2nsgx3mx"; -}; -linux_py_36_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp36-cp36m-linux_x86_64.whl"; - sha256 = "0ypkp8cfhharsyyikb1qgf44cfm6284km9xswzvzymjzz75vg3gd"; -}; -linux_py_37_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp37-cp37m-linux_x86_64.whl"; - sha256 = "0virp8nn2ysx4855hq29kas6fm6b3dsiybwzdxy9nnb9n2d8qlm2"; -}; -mac_py_2_cpu = { - url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py2-none-any.whl"; - sha256 = "14f86k3pgq7z6i4s4im55zpp38f0drnm7xlclavsgcc0nxnj3z26"; -}; -mac_py_3_cpu = { - url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py3-none-any.whl"; - sha256 = "0f3swpcjfgqhj6h5wnx8snc0xjkx4hnkqx83fmlrwpncs8c131d3"; -}; -} -- cgit 1.4.1