diff options
Diffstat (limited to 'nixpkgs/pkgs/development/python-modules/tensorflow')
4 files changed, 319 insertions, 0 deletions
diff --git a/nixpkgs/pkgs/development/python-modules/tensorflow/bin.nix b/nixpkgs/pkgs/development/python-modules/tensorflow/bin.nix new file mode 100644 index 000000000000..03f1e66bf501 --- /dev/null +++ b/nixpkgs/pkgs/development/python-modules/tensorflow/bin.nix @@ -0,0 +1,93 @@ +{ stdenv +, lib +, fetchurl +, buildPythonPackage +, isPy3k, isPy36, pythonOlder +, astor +, gast +, numpy +, six +, termcolor +, protobuf +, absl-py +, grpcio +, mock +, backports_weakref +, enum34 +, tensorflow-estimator +, tensorflow-tensorboard +, cudaSupport ? false +, cudatoolkit ? null +, cudnn ? null +, nvidia_x11 ? null +, zlib +, python +, symlinkJoin +, keras-applications +, keras-preprocessing +}: + +# We keep this binary build for two reasons: +# - the source build doesn't work on Darwin. +# - the source build is currently brittle and not easy to maintain + +assert cudaSupport -> cudatoolkit != null + && cudnn != null + && nvidia_x11 != null; +let + cudatoolkit_joined = symlinkJoin { + name = "unsplit_cudatoolkit"; + paths = [ cudatoolkit.out + cudatoolkit.lib ];}; + +in buildPythonPackage rec { + pname = "tensorflow"; + version = "1.13.1"; + format = "wheel"; + + src = let + pyVerNoDot = lib.strings.stringAsChars (x: if x == "." then "" else x) "${python.pythonVersion}"; + pyver = if stdenv.isDarwin then builtins.substring 0 1 pyVerNoDot else pyVerNoDot; + platform = if stdenv.isDarwin then "mac" else "linux"; + unit = if cudaSupport then "gpu" else "cpu"; + key = "${platform}_py_${pyver}_${unit}"; + dls = import (./. + "/tf${version}-hashes.nix"); + in fetchurl dls.${key}; + + propagatedBuildInputs = [ protobuf numpy termcolor grpcio six astor absl-py gast tensorflow-estimator tensorflow-tensorboard keras-applications keras-preprocessing ] + ++ lib.optional (!isPy3k) mock + ++ lib.optionals (pythonOlder "3.4") [ backports_weakref ]; + + # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow + # and the propageted input tensorflow-tensorboard which causes environment collisions. + # another possibility would be to have tensorboard only in the buildInputs + # https://github.com/tensorflow/tensorflow/blob/v1.7.1/tensorflow/tools/pip_package/setup.py#L79 + postInstall = '' + rm $out/bin/tensorboard + ''; + + # Note that we need to run *after* the fixup phase because the + # libraries are loaded at runtime. If we run in preFixup then + # patchelf --shrink-rpath will remove the cuda libraries. + postFixup = let + rpath = stdenv.lib.makeLibraryPath + ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit_joined cudnn nvidia_x11 ]); + in + lib.optionalString (stdenv.isLinux) '' + rrPath="$out/${python.sitePackages}/tensorflow/:$out/${python.sitePackages}/tensorflow/contrib/tensor_forest/:${rpath}" + internalLibPath="$out/${python.sitePackages}/tensorflow/python/_pywrap_tensorflow_internal.so" + find $out -name '*${stdenv.hostPlatform.extensions.sharedLibrary}' -exec patchelf --set-rpath "$rrPath" {} \; + ''; + + + meta = with stdenv.lib; { + description = "Computation using data flow graphs for scalable machine learning"; + homepage = http://tensorflow.org; + license = licenses.asl20; + maintainers = with maintainers; [ jyp abbradar ]; + platforms = with platforms; linux ++ lib.optionals (!cudaSupport) darwin; + # Python 2.7 build uses different string encoding. + # See https://github.com/NixOS/nixpkgs/pull/37044#issuecomment-373452253 + broken = stdenv.isDarwin && !isPy3k; + }; +} diff --git a/nixpkgs/pkgs/development/python-modules/tensorflow/default.nix b/nixpkgs/pkgs/development/python-modules/tensorflow/default.nix new file mode 100644 index 000000000000..8163243eb0a4 --- /dev/null +++ b/nixpkgs/pkgs/development/python-modules/tensorflow/default.nix @@ -0,0 +1,154 @@ +{ stdenv, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin +, buildPythonPackage, isPy3k, pythonOlder, pythonAtLeast +, which, swig, binutils, glibcLocales +, python, jemalloc, openmpi +, numpy, six, protobuf, tensorflow-tensorboard, backports_weakref, mock, enum34, absl-py +, cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null +# XLA without CUDA is broken +, xlaSupport ? cudaSupport +# Default from ./configure script +, cudaCapabilities ? [ "3.5" "5.2" ] +, sse42Support ? false +, avx2Support ? false +, fmaSupport ? false +}: + +assert cudaSupport -> nvidia_x11 != null + && cudatoolkit != null + && cudnn != null; + +# unsupported combination +assert ! (stdenv.isDarwin && cudaSupport); + +let + + withTensorboard = pythonOlder "3.6"; + + cudatoolkit_joined = symlinkJoin { + name = "${cudatoolkit.name}-unsplit"; + paths = [ cudatoolkit.out cudatoolkit.lib ]; + }; + + tfFeature = x: if x then "1" else "0"; + + version = "1.5.0"; + + pkg = buildBazelPackage rec { + name = "tensorflow-build-${version}"; + + src = fetchFromGitHub { + owner = "tensorflow"; + repo = "tensorflow"; + rev = "v${version}"; + sha256 = "1c4djsaip901nasm7a6dsimr02bsv70a7b1g0kysb4n39qpdh22q"; + }; + + patches = [ + # Fix build with Bazel >= 0.10 + (fetchpatch { + url = "https://github.com/tensorflow/tensorflow/commit/6fcfab770c2672e2250e0f5686b9545d99eb7b2b.patch"; + sha256 = "0p61za1mx3a7gj1s5lsps16fcw18iwnvq2b46v1kyqfgq77a12vb"; + }) + (fetchpatch { + url = "https://github.com/tensorflow/tensorflow/commit/3f57956725b553d196974c9ad31badeb3eabf8bb.patch"; + sha256 = "11dja5gqy0qw27sc9b6yw9r0lfk8dznb32vrqqfcnypk2qmv26va"; + }) + ]; + + nativeBuildInputs = [ swig which ]; + + buildInputs = [ python jemalloc openmpi glibcLocales numpy ] + ++ lib.optionals cudaSupport [ cudatoolkit cudnn nvidia_x11 ]; + + preConfigure = '' + patchShebangs configure + + export PYTHON_BIN_PATH="${python.interpreter}" + export PYTHON_LIB_PATH="$NIX_BUILD_TOP/site-packages" + export TF_NEED_GCP=1 + export TF_NEED_HDFS=1 + export TF_ENABLE_XLA=${tfFeature xlaSupport} + export CC_OPT_FLAGS=" " + # https://github.com/tensorflow/tensorflow/issues/14454 + export TF_NEED_MPI=${tfFeature cudaSupport} + export TF_NEED_CUDA=${tfFeature cudaSupport} + ${lib.optionalString cudaSupport '' + export CUDA_TOOLKIT_PATH=${cudatoolkit_joined} + export TF_CUDA_VERSION=${cudatoolkit.majorVersion} + export CUDNN_INSTALL_PATH=${cudnn} + export TF_CUDNN_VERSION=${cudnn.majorVersion} + export GCC_HOST_COMPILER_PATH=${cudatoolkit.cc}/bin/gcc + export TF_CUDA_COMPUTE_CAPABILITIES=${lib.concatStringsSep "," cudaCapabilities} + ''} + + mkdir -p "$PYTHON_LIB_PATH" + ''; + + NIX_LDFLAGS = lib.optionals cudaSupport [ "-lcublas" "-lcudnn" "-lcuda" "-lcudart" ]; + + hardeningDisable = [ "all" ]; + + bazelFlags = [ "--config=opt" ] + ++ lib.optional sse42Support "--copt=-msse4.2" + ++ lib.optional avx2Support "--copt=-mavx2" + ++ lib.optional fmaSupport "--copt=-mfma" + ++ lib.optional cudaSupport "--config=cuda"; + + bazelTarget = "//tensorflow/tools/pip_package:build_pip_package"; + + fetchAttrs = { + preInstall = '' + rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker,local_*,\@local_*} + ''; + + sha256 = "1nc98aqrp14q7llypcwaa0kdn9xi7r0p1mnd3vmmn1m299py33ca"; + }; + + buildAttrs = { + preBuild = '' + patchShebangs . + find -type f -name CROSSTOOL\* -exec sed -i \ + -e 's,/usr/bin/ar,${binutils.bintools}/bin/ar,g' \ + {} \; + ''; + + installPhase = '' + sed -i 's,.*bdist_wheel.*,cp -rL . "$out"; exit 0,' bazel-bin/tensorflow/tools/pip_package/build_pip_package + bazel-bin/tensorflow/tools/pip_package/build_pip_package $PWD/dist + ''; + }; + + dontFixup = true; + }; + +in buildPythonPackage rec { + pname = "tensorflow"; + inherit version; + + src = pkg; + + installFlags = lib.optional (!withTensorboard) "--no-dependencies"; + + postPatch = lib.optionalString (pythonAtLeast "3.4") '' + sed -i '/enum34/d' setup.py + ''; + + propagatedBuildInputs = [ numpy six protobuf absl-py ] + ++ lib.optional (!isPy3k) mock + ++ lib.optionals (pythonOlder "3.4") [ backports_weakref enum34 ] + ++ lib.optional withTensorboard tensorflow-tensorboard; + + # Actual tests are slow and impure. + checkPhase = '' + ${python.interpreter} -c "import tensorflow" + ''; + + meta = with stdenv.lib; { + description = "Computation using data flow graphs for scalable machine learning"; + homepage = http://tensorflow.org; + license = licenses.asl20; + maintainers = with maintainers; [ jyp abbradar ]; + platforms = platforms.linux; + broken = !(xlaSupport -> cudaSupport); + }; +} diff --git a/nixpkgs/pkgs/development/python-modules/tensorflow/prefetcher.sh b/nixpkgs/pkgs/development/python-modules/tensorflow/prefetcher.sh new file mode 100644 index 000000000000..d4ec2e757b2a --- /dev/null +++ b/nixpkgs/pkgs/development/python-modules/tensorflow/prefetcher.sh @@ -0,0 +1,30 @@ +version=1.13.1 +hashfile=tf${version}-hashes.nix +rm -f $hashfile +echo "{" >> $hashfile +for sys in "linux" "mac"; do + for tfpref in "cpu/tensorflow" "gpu/tensorflow_gpu"; do + for pykind in "py2-none-any" "py3-none-any" "cp27-none-linux_x86_64" "cp35-cp35m-linux_x86_64" "cp36-cp36m-linux_x86_64" "cp37-cp37m-linux_x86_64"; do + if [ $sys == "mac" ]; then + [[ $pykind =~ py.* ]] && [[ $tfpref =~ cpu.* ]] + result=$? + pyver=${pykind:2:1} + flavour=cpu + else + [[ $pykind =~ .*linux.* ]] + result=$? + pyver=${pykind:2:2} + flavour=${tfpref:0:3} + fi + if [ $result == 0 ]; then + url=https://storage.googleapis.com/tensorflow/$sys/$tfpref-$version-$pykind.whl + hash=$(nix-prefetch-url $url) + echo "${sys}_py_${pyver}_${flavour} = {" >> $hashfile + echo " url = \"$url\";" >> $hashfile + echo " sha256 = \"$hash\";" >> $hashfile + echo "};" >> $hashfile + fi + done + done +done +echo "}" >> $hashfile diff --git a/nixpkgs/pkgs/development/python-modules/tensorflow/tf1.13.1-hashes.nix b/nixpkgs/pkgs/development/python-modules/tensorflow/tf1.13.1-hashes.nix new file mode 100644 index 000000000000..bbfb0632c3c5 --- /dev/null +++ b/nixpkgs/pkgs/development/python-modules/tensorflow/tf1.13.1-hashes.nix @@ -0,0 +1,42 @@ +{ +linux_py_27_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.13.1-cp27-none-linux_x86_64.whl"; + sha256 = "0y1vd3y5fxcjj5d35qbk8482b0s642nyp0c2sm068vx5wd4sjpcg"; +}; +linux_py_35_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.13.1-cp35-cp35m-linux_x86_64.whl"; + sha256 = "0b27swk4c2vaimwzbzl4c7xnccr9cfak5a3848lfqlcavcmbp94j"; +}; +linux_py_36_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl"; + sha256 = "087jwjby3bym09z55cjhc587aasf01y6l009p1q2vcpfq7s7ljmk"; +}; +linux_py_37_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.13.1-cp37-cp37m-linux_x86_64.whl"; + sha256 = "0as68dp87lh7ffcccb149km6vws15ap04604irxwz35fq9h7grxg"; +}; +linux_py_27_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.13.1-cp27-none-linux_x86_64.whl"; + sha256 = "0bf239f2bnsbqs3qh4xdql9pgbsm0zk7j8q1hg0wn0wrq440n0ds"; +}; +linux_py_35_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.13.1-cp35-cp35m-linux_x86_64.whl"; + sha256 = "1cqav22a8yz6fzk46z6kv1ha2i28h5wccbd7k66drrfxibmb93j0"; +}; +linux_py_36_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.13.1-cp36-cp36m-linux_x86_64.whl"; + sha256 = "1xnbiz36z7nicqrv0cmymfnwb8mdz2hifcv71gh6gnyi1962f2d7"; +}; +linux_py_37_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.13.1-cp37-cp37m-linux_x86_64.whl"; + sha256 = "10gcrmd9y5a89wpi4rpp9scc9l2krijv8yjp7iphlykmn54ps74k"; +}; +mac_py_2_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.13.1-py2-none-any.whl"; + sha256 = "1a6y5xj2wqkd8qmabn2xjg3q7x2jfixwrz351dgcxlhy8qy5yc0g"; +}; +mac_py_3_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.13.1-py3-none-any.whl"; + sha256 = "1klsv18k0isfd61z1wirfz1lnqmx8k73ga8g9s18yand65iycads"; +}; +} |