diff options
3 files changed, 155 insertions, 78 deletions
diff --git a/pkgs/development/python-modules/tensorflow/bin.nix b/pkgs/development/python-modules/tensorflow/bin.nix index 38fec9adc7dc..e81dc2dbd002 100644 --- a/pkgs/development/python-modules/tensorflow/bin.nix +++ b/pkgs/development/python-modules/tensorflow/bin.nix @@ -14,6 +14,9 @@ , absl-py , grpcio , mock +, scipy +, wheel +, opt-einsum , backports_weakref , tensorflow-estimator , tensorflow-tensorboard @@ -55,21 +58,22 @@ in buildPythonPackage { src = let pyVerNoDot = lib.strings.stringAsChars (x: if x == "." then "" else x) python.pythonVersion; - pyver = if stdenv.isDarwin then builtins.substring 0 1 pyVerNoDot else pyVerNoDot; platform = if stdenv.isDarwin then "mac" else "linux"; unit = if cudaSupport then "gpu" else "cpu"; - key = "${platform}_py_${pyver}_${unit}"; + key = "${platform}_py_${pyVerNoDot}_${unit}"; in fetchurl packages.${key}; propagatedBuildInputs = [ protobuf numpy + scipy termcolor grpcio six astor absl-py gast + opt-einsum google-pasta wrapt tensorflow-estimator @@ -79,40 +83,94 @@ in buildPythonPackage { ] ++ lib.optional (!isPy3k) mock ++ lib.optionals (pythonOlder "3.4") [ backports_weakref ]; - nativeBuildInputs = lib.optional cudaSupport addOpenGLRunpath; + nativeBuildInputs = [ wheel ] ++ lib.optional cudaSupport addOpenGLRunpath; - # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow - # and the propageted input tensorflow-tensorboard which causes environment collisions. - # another possibility would be to have tensorboard only in the buildInputs - # https://github.com/tensorflow/tensorflow/blob/v1.7.1/tensorflow/tools/pip_package/setup.py#L79 - postInstall = '' - rm $out/bin/tensorboard + preConfigure = '' + unset SOURCE_DATE_EPOCH + + # Make sure that dist and the wheel file are writable. + chmod u+rwx -R ./dist + + pushd dist + + # Unpack the wheel file. + wheel unpack --dest unpacked ./*.whl + + # Tensorflow has a hard dependency on gast==0.2.2, but we relax it to + # gast==0.3.2. + substituteInPlace ./unpacked/tensorflow*/tensorflow_core/tools/pip_package/setup.py --replace "gast == 0.2.2" "gast == 0.3.2" + substituteInPlace ./unpacked/tensorflow*/tensorflow_*.dist-info/METADATA --replace "gast (==0.2.2)" "gast (==0.3.2)" + + # Pack the wheel file back up. + wheel pack ./unpacked/tensorflow* + + popd ''; # Note that we need to run *after* the fixup phase because the # libraries are loaded at runtime. If we run in preFixup then # patchelf --shrink-rpath will remove the cuda libraries. - postFixup = let - rpath = stdenv.lib.makeLibraryPath - ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]); - in - lib.optionalString stdenv.isLinux '' - rrPath="$out/${python.sitePackages}/tensorflow/:$out/${python.sitePackages}/tensorflow/contrib/tensor_forest/:${rpath}" - internalLibPath="$out/${python.sitePackages}/tensorflow/python/_pywrap_tensorflow_internal.so" - find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do - patchelf --set-rpath "$rrPath" "$lib" - ${lib.optionalString cudaSupport '' - addOpenGLRunpath "$lib" - ''} - done - ''; + postFixup = + let + # rpaths we only need to add if CUDA is enabled. + cudapaths = lib.optionals cudaSupport [ + cudatoolkit.out + cudatoolkit.lib + cudnn + nvidia_x11 + ]; + + libpaths = [ + stdenv.cc.cc.lib + zlib + ]; + + rpath = stdenv.lib.makeLibraryPath (libpaths ++ cudapaths); + in + lib.optionalString stdenv.isLinux '' + # This is an array containing all the directories in the tensorflow2 + # package that contain .so files. + # + # TODO: Create this list programmatically, and remove paths that aren't + # actually needed. + rrPathArr=( + "$out/${python.sitePackages}/tensorflow_core/" + "$out/${python.sitePackages}/tensorflow_core/compiler/tf2tensorrt/" + "$out/${python.sitePackages}/tensorflow_core/compiler/tf2xla/ops/" + "$out/${python.sitePackages}/tensorflow_core/lite/experimental/microfrontend/python/ops/" + "$out/${python.sitePackages}/tensorflow_core/lite/python/interpreter_wrapper/" + "$out/${python.sitePackages}/tensorflow_core/lite/python/optimize/" + "$out/${python.sitePackages}/tensorflow_core/python/" + "$out/${python.sitePackages}/tensorflow_core/python/framework/" + "${rpath}" + ) + + # The the bash array into a colon-separated list of RPATHs. + rrPath=$(IFS=$':'; echo "''${rrPathArr[*]}") + echo "about to run patchelf with the following rpath: $rrPath" + + find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do + echo "about to patchelf $lib..." + chmod a+rx "$lib" + patchelf --set-rpath "$rrPath" "$lib" + ${lib.optionalString cudaSupport '' + addOpenGLRunpath "$lib" + ''} + done + ''; + pythonImportsCheck = [ + "tensorflow" + "tensorflow.keras" + "tensorflow.python" + "tensorflow.python.framework" + ]; meta = with stdenv.lib; { description = "Computation using data flow graphs for scalable machine learning"; homepage = http://tensorflow.org; license = licenses.asl20; - maintainers = with maintainers; [ jyp abbradar ]; + maintainers = with maintainers; [ jyp abbradar cdepillabout ]; platforms = [ "x86_64-linux" "x86_64-darwin" ]; # Python 2.7 build uses different string encoding. # See https://github.com/NixOS/nixpkgs/pull/37044#issuecomment-373452253 diff --git a/pkgs/development/python-modules/tensorflow/binary-hashes.nix b/pkgs/development/python-modules/tensorflow/binary-hashes.nix index fa4809dc3f19..90848e93273f 100644 --- a/pkgs/development/python-modules/tensorflow/binary-hashes.nix +++ b/pkgs/development/python-modules/tensorflow/binary-hashes.nix @@ -1,43 +1,51 @@ { -version = "1.14.0"; +version = "2.1.0"; +linux_py_27_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.1.0-cp27-cp27mu-manylinux2010_x86_64.whl"; + sha256 = "17lnhr7vdrls68c79n3sah5rpd0q1x2v5m84azvlyxxh2wpypfmb"; +}; linux_py_27_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp27-none-linux_x86_64.whl"; - sha256 = "0yywdrfk97dh1bxhibspg0raz70fx9lcczj6xlimqy4xb60clx7k"; + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp27-cp27mu-manylinux2010_x86_64.whl"; + sha256 = "10lz3i4pcpgqrcbjmxm0n7k1gsqlpna3kdid902j2fy060cpi93z"; +}; +linux_py_35_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.1.0-cp35-cp35m-manylinux2010_x86_64.whl"; + sha256 = "09s081n08dpmflwgir3zwzfijfpmahbh2gy5fn5bv5ll86g1szsy"; }; linux_py_35_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp35-cp35m-linux_x86_64.whl"; - sha256 = "1xvyb6xcrjhlwvrmrhn5vs9xy7g98smqmpv4i3hhpry4qyasphhj"; + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp35-cp35m-manylinux2010_x86_64.whl"; + sha256 = "1aa7v9fnvx03hqvhl3x3xcn41qy6qxw5xybg54ifjvvicp455c8l"; }; -linux_py_36_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl"; - sha256 = "1psd9vyxz9f39dwj77nvrg373sxv3p5vdp9fnz81dpsm0b0mwl44"; +linux_py_36_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl"; + sha256 = "1dqp080ljbl9v3115vjp63ls0fimiwym6zxyanyhrlk8kwsq20zc"; }; -linux_py_37_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp37-cp37m-linux_x86_64.whl"; - sha256 = "0bg2sb1n2ag27r7ww695kg5hb0mjrw4kc5893krmixx2j71860c5"; +linux_py_36_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl"; + sha256 = "133z8anx7xm9rr5i9s9dwnp1wf06nr6s7q1lbs4lxpk6kn9nl480"; }; -linux_py_27_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp27-none-linux_x86_64.whl"; - sha256 = "0y1x91gayg6pjddgl8ndcm63wfzhyv4s5khgl7ffzsgni1ivaqw5"; +linux_py_37_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.1.0-cp37-cp37m-manylinux2010_x86_64.whl"; + sha256 = "0yabl3xmcpr67w0zksqs3qc68nl9ax0vcd7w7b35nq8f65xl0ghy"; }; -linux_py_35_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp35-cp35m-linux_x86_64.whl"; - sha256 = "03piggpbz1jx8m2b95spq3jrdff4w6xx63ji07am7hyw2nsgx3mx"; +linux_py_37_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp37-cp37m-manylinux2010_x86_64.whl"; + sha256 = "04gngbngyg7p1gwx1q89my0cl8j7lq4kknqh51s2ynrix71zvsy6"; }; -linux_py_36_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp36-cp36m-linux_x86_64.whl"; - sha256 = "0ypkp8cfhharsyyikb1qgf44cfm6284km9xswzvzymjzz75vg3gd"; +mac_py_27_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.1.0-cp27-cp27m-macosx_10_9_x86_64.whl"; + sha256 = "1mprp72w5kk0lyjm2mh4lf57827xk3wsg28c4gizwm00ydfgacg6"; }; -linux_py_37_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp37-cp37m-linux_x86_64.whl"; - sha256 = "0virp8nn2ysx4855hq29kas6fm6b3dsiybwzdxy9nnb9n2d8qlm2"; +mac_py_35_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.1.0-cp35-cp35m-macosx_10_6_intel.whl"; + sha256 = "1as7brf5ai6r7v1di9646jfrbnirpk2b0d1g29mn3shavb62kw8w"; }; -mac_py_2_cpu = { - url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py2-none-any.whl"; - sha256 = "14f86k3pgq7z6i4s4im55zpp38f0drnm7xlclavsgcc0nxnj3z26"; +mac_py_36_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.1.0-cp36-cp36m-macosx_10_9_x86_64.whl"; + sha256 = "1v1rw9kjrskhcq1yas4ly2yfnzf2i1pjh6qg6zixfbkpkw7sw3wc"; }; -mac_py_3_cpu = { - url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py3-none-any.whl"; - sha256 = "0f3swpcjfgqhj6h5wnx8snc0xjkx4hnkqx83fmlrwpncs8c131d3"; +mac_py_37_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.1.0-cp37-cp37m-macosx_10_9_x86_64.whl"; + sha256 = "1hh4n0d97mrq35cmmsrnlmcv9vlswsyjy368lj3pda3y9dvck3rf"; }; } diff --git a/pkgs/development/python-modules/tensorflow/prefetcher.sh b/pkgs/development/python-modules/tensorflow/prefetcher.sh index d590fb0f1732..abb0faac2834 100755 --- a/pkgs/development/python-modules/tensorflow/prefetcher.sh +++ b/pkgs/development/python-modules/tensorflow/prefetcher.sh @@ -1,33 +1,44 @@ #!/usr/bin/env bash -version=1.14.0 +version=2.1.0 + +# List of binary wheels for Tensorflow. The most recent versions can be found +# on the following page: +# https://www.tensorflow.org/install/pip?lang=python3#package-location +url_and_key_list=( + "linux_py_27_gpu https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${version}-cp27-cp27mu-manylinux2010_x86_64.whl" + "linux_py_27_cpu https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-${version}-cp27-cp27mu-manylinux2010_x86_64.whl" + "linux_py_35_gpu https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${version}-cp35-cp35m-manylinux2010_x86_64.whl" + "linux_py_35_cpu https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-${version}-cp35-cp35m-manylinux2010_x86_64.whl" + "linux_py_36_gpu https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${version}-cp36-cp36m-manylinux2010_x86_64.whl" + "linux_py_36_cpu https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-${version}-cp36-cp36m-manylinux2010_x86_64.whl" + "linux_py_37_gpu https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${version}-cp37-cp37m-manylinux2010_x86_64.whl" + "linux_py_37_cpu https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-${version}-cp37-cp37m-manylinux2010_x86_64.whl" + "mac_py_27_cpu https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-cp27-cp27m-macosx_10_9_x86_64.whl" + "mac_py_35_cpu https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-cp35-cp35m-macosx_10_6_intel.whl" + "mac_py_36_cpu https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-cp36-cp36m-macosx_10_9_x86_64.whl" + "mac_py_37_cpu https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-cp37-cp37m-macosx_10_9_x86_64.whl" +) + hashfile=binary-hashes.nix rm -f $hashfile echo "{" >> $hashfile echo "version = \"$version\";" >> $hashfile -for sys in "linux" "mac"; do - for tfpref in "cpu/tensorflow" "gpu/tensorflow_gpu"; do - for pykind in "py2-none-any" "py3-none-any" "cp27-none-linux_x86_64" "cp35-cp35m-linux_x86_64" "cp36-cp36m-linux_x86_64" "cp37-cp37m-linux_x86_64"; do - if [ $sys == "mac" ]; then - [[ $pykind =~ py.* ]] && [[ $tfpref =~ cpu.* ]] - result=$? - pyver=${pykind:2:1} - flavour=cpu - else - [[ $pykind =~ .*linux.* ]] - result=$? - pyver=${pykind:2:2} - flavour=${tfpref:0:3} - fi - if [ $result == 0 ]; then - url=https://storage.googleapis.com/tensorflow/$sys/$tfpref-$version-$pykind.whl - hash=$(nix-prefetch-url $url) - echo "${sys}_py_${pyver}_${flavour} = {" >> $hashfile - echo " url = \"$url\";" >> $hashfile - echo " sha256 = \"$hash\";" >> $hashfile - echo "};" >> $hashfile - fi - done - done + +for url_and_key in "${url_and_key_list[@]}"; do + key=$(echo "$url_and_key" | cut -d' ' -f1) + url=$(echo "$url_and_key" | cut -d' ' -f2) + + echo "prefetching ${url}..." + hash=$(nix-prefetch-url $url) + + echo "$key = {" >> $hashfile + echo " url = \"$url\";" >> $hashfile + echo " sha256 = \"$hash\";" >> $hashfile + echo "};" >> $hashfile + + echo done + echo "}" >> $hashfile +echo "done." |