From 3df4e2df9566226fc9c1cedf3cd19f95f2a2679b Mon Sep 17 00:00:00 2001
From: Timo Kaufmann <timokau@zoho.com>
Date: Thu, 20 Jun 2019 20:17:13 +0200
Subject: tensorflow: re-enable build from source [WIP]

---
 .../python-modules/tensorflow/default.nix          | 290 ++++++++++++++++++---
 .../python-modules/tensorflow/no-saved-proto.patch |  14 +
 pkgs/top-level/python-packages.nix                 |   8 +-
 3 files changed, 267 insertions(+), 45 deletions(-)
 create mode 100644 pkgs/development/python-modules/tensorflow/no-saved-proto.patch

diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix
index a8f902fdb38f..98e651a5d95c 100644
--- a/pkgs/development/python-modules/tensorflow/default.nix
+++ b/pkgs/development/python-modules/tensorflow/default.nix
@@ -1,9 +1,42 @@
 { stdenv, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin
+, git
 , buildPythonPackage, isPy3k, pythonOlder, pythonAtLeast
-, which, swig, binutils, glibcLocales
+, which, binutils, glibcLocales
 , python, jemalloc, openmpi
-, numpy, six, protobuf, tensorflow-tensorboard, backports_weakref, mock, enum34, absl-py
-, cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null
+, numpy, tensorflow-tensorboard, backports_weakref, mock, enum34, absl-py
+, future
+, keras-preprocessing
+, keras-applications
+, astor
+, gast
+, google-pasta
+, termcolor
+, cython
+, flatbuffers
+, giflib
+, libjpeg
+, grpc
+, grpcio
+, hwloc
+, icu
+, jsoncpp
+, lmdb
+, nasm
+, sqlite
+, pcre
+, libpng
+, six
+, snappy
+, swig
+, wrapt
+, zlib
+, protobuf
+, protobuf_cc
+, curl
+, tensorflow-estimator
+, setuptools
+, wheel
+, cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null, nccl ? null
 # XLA without CUDA is broken
 , xlaSupport ? cudaSupport
 # Default from ./configure script
@@ -21,7 +54,6 @@ assert cudaSupport -> nvidia_x11 != null
 assert ! (stdenv.isDarwin && cudaSupport);
 
 let
-
   withTensorboard = pythonOlder "3.6";
 
   cudatoolkit_joined = symlinkJoin {
@@ -29,40 +61,147 @@ let
     paths = [ cudatoolkit.out cudatoolkit.lib ];
   };
 
+  bazel_cuda_toolkit = symlinkJoin {
+    name = "bazel-gcc-toolkit";
+    paths = [
+      cudatoolkit.cc
+      cudatoolkit.out
+      cudatoolkit.lib
+      binutils.bintools # for ar, dwp, nm, objcopy, objdump, strip
+    ];
+  };
+
   tfFeature = x: if x then "1" else "0";
 
-  version = "1.5.0";
+  version = "1.14";
+  variant = if cudaSupport then "_gpu" else "";
+  pname = "tensorflow${variant}";
 
-  pkg = buildBazelPackage rec {
-    name = "tensorflow-build-${version}";
+  bazel-wheel = buildBazelPackage rec {
+    # indicate which configuration of the wheel is being built
+    name = let
+      python_tag = with lib.versions; "cp${major python.version}${minor python.version}"; # cp37
+      arch_platform = lib.splitString "-" stdenv.hostPlatform.system;
+      platform = lib.last arch_platform; # "linux"
+      arch = lib.head arch_platform; # "x86_64"
+    in
+      # https://www.python.org/dev/peps/pep-0427/#file-name-convention
+      # tensorflow_gpu-1.14.0-cp27-none-linux_x86_64.whl
+      "${pname}-${version}-${python_tag}-none-${platform}_${arch}.whl";
 
     src = fetchFromGitHub {
       owner = "tensorflow";
       repo = "tensorflow";
-      rev = "v${version}";
-      sha256 = "1c4djsaip901nasm7a6dsimr02bsv70a7b1g0kysb4n39qpdh22q";
+      rev = "r${version}";
+      sha256 = "071rndqh3xy4y5idlykqv656lggp25vy4qx1divz4id3im1zlr13";
     };
 
     patches = [
-      # Fix build with Bazel >= 0.10
+      # Work around https://github.com/tensorflow/tensorflow/issues/24752
+      ./no-saved-proto.patch
+
+      # https://github.com/tensorflow/tensorflow/pull/29673
       (fetchpatch {
-        url = "https://github.com/tensorflow/tensorflow/commit/6fcfab770c2672e2250e0f5686b9545d99eb7b2b.patch";
-        sha256 = "0p61za1mx3a7gj1s5lsps16fcw18iwnvq2b46v1kyqfgq77a12vb";
+        name = "fix-compile-with-cuda-and-mpi.patch";
+        url = "https://github.com/tensorflow/tensorflow/pull/29673/commits/498e35a3bfe38dd75cf1416a1a23c07c3b59e6af.patch";
+        sha256 = "1m2qmwv1ysqa61z6255xggwbq6mnxbig749bdvrhnch4zydxb4di";
       })
+
+      # https://github.com/tensorflow/tensorflow/issues/29220
       (fetchpatch {
-        url = "https://github.com/tensorflow/tensorflow/commit/3f57956725b553d196974c9ad31badeb3eabf8bb.patch";
-        sha256 = "11dja5gqy0qw27sc9b6yw9r0lfk8dznb32vrqqfcnypk2qmv26va";
+        name = "bazel-0.27.patch";
+        url = "https://github.com/tensorflow/tensorflow/commit/cfccbdb8c4a92dd26382419dceb4d934c2380391.patch";
+        sha256 = "1l56wjia2c4685flsfkkgy471wx3c66wyv8khspv06zchj0k0liw";
       })
     ];
 
+    # On update, it can be useful to steal the changes from gentoo
+    # https://gitweb.gentoo.org/repo/gentoo.git/tree/sci-libs/tensorflow
+
     nativeBuildInputs = [ swig which ];
 
-    buildInputs = [ python jemalloc openmpi glibcLocales numpy ]
-      ++ lib.optionals cudaSupport [ cudatoolkit cudnn nvidia_x11 ];
+    buildInputs = [
+      python
+      jemalloc
+      openmpi
+      glibcLocales
+      git
+
+      # python deps needed during wheel build time
+      numpy
+      keras-preprocessing
+
+      # libs taken from system through the TF_SYS_LIBS mechanism
+      absl-py
+
+      # for building the wheel
+      setuptools
+      wheel
+    ] ++ lib.optionals (!isPy3k) [
+      future
+      mock
+    ] ++ lib.optionals cudaSupport [
+      cudatoolkit
+      cudnn
+      nvidia_x11
+    ];
+
+    # Take as many libraries from the system as possible. Keep in sync with
+    # list of valid syslibs in
+    # https://github.com/perfinion/tensorflow/blob/master/third_party/systemlibs/syslibs_configure.bzl
+    SYSLIBS= [
+      "absl_py"
+      "astor_archive"
+      "boringssl"
+      "com_github_googleapis_googleapis"
+      "com_github_googlecloudplatform_google_cloud_cpp"
+      "com_google_protobuf"
+      "com_google_protobuf_cc"
+      "com_googlesource_code_re2"
+      "curl"
+      "cython"
+      "double_conversion"
+      "enum34_archive"
+      "flatbuffers"
+      "gast_archive"
+      "gif_archive"
+      "grpc"
+      "hwloc"
+      "icu"
+      "jpeg"
+      "jsoncpp_git"
+      "keras_applications_archive"
+      "lmdb"
+      "nasm"
+      # "nsync" # not packaged in nixpkgs
+      "sqlite"
+      "pasta"
+      "pcre"
+      "png_archive"
+      "protobuf_archive"
+      "six_archive"
+      "snappy"
+      "swig"
+      "termcolor_archive"
+      "wrapt"
+      "zlib_archive"
+    ];
 
     preConfigure = ''
       patchShebangs configure
 
+      # dummy ldconfig
+      mkdir dummy-ldconfig
+      echo "#!${stdenv.shell}" > dummy-ldconfig/ldconfig
+      chmod +x dummy-ldconfig/ldconfig
+      export PATH="$PWD/dummy-ldconfig:$PATH"
+
+      # arbitrarily set to the current latest bazel version, overly careful
+      export TF_IGNORE_MAX_BAZEL_VERSION=1
+
+      # don't rebuild the world
+      export TF_SYSTEM_LIBS=${lib.concatStringsSep " " SYSLIBS}
+
       export PYTHON_BIN_PATH="${python.interpreter}"
       export PYTHON_LIB_PATH="$NIX_BUILD_TOP/site-packages"
       export TF_NEED_GCP=1
@@ -73,26 +212,43 @@ let
       export TF_NEED_MPI=${tfFeature cudaSupport}
       export TF_NEED_CUDA=${tfFeature cudaSupport}
       ${lib.optionalString cudaSupport ''
-        export CUDA_TOOLKIT_PATH=${cudatoolkit_joined}
+        export TF_CUDA_PATHS="${cudatoolkit_joined},${cudnn},${nccl}"
         export TF_CUDA_VERSION=${cudatoolkit.majorVersion}
-        export CUDNN_INSTALL_PATH=${cudnn}
         export TF_CUDNN_VERSION=${cudnn.majorVersion}
-        export GCC_HOST_COMPILER_PATH=${cudatoolkit.cc}/bin/gcc
+        export GCC_HOST_COMPILER_PATH=${bazel_cuda_toolkit}/bin/gcc
+        export GCC_HOST_COMPILER_PREFIX="${bazel_cuda_toolkit}/bin"
         export TF_CUDA_COMPUTE_CAPABILITIES=${lib.concatStringsSep "," cudaCapabilities}
       ''}
 
+      # https://github.com/tensorflow/tensorflow/issues/20919
+      sed -i '/androidndk/d' tensorflow/lite/kernels/internal/BUILD
+
       mkdir -p "$PYTHON_LIB_PATH"
     '';
 
+    # ar, cpp, gcov, strip, gcc, compat-ld, ld, objdump, dwp, nm, objcopy
+    configurePhase = ''
+      runHook preConfigure
+      # no flags (options provided by previously set environment variables)
+      export AR="${binutils.bintools}/bin/ar"
+      export BAZEL_TOOLS_OVERRIDE="ar=${binutils.bintools}/bin/ar"
+      # crosstool used with cuda
+      ./configure
+
+      runHook postConfigure
+    '';
+
+    # FIXME
     NIX_LDFLAGS = lib.optionals cudaSupport [ "-lcublas" "-lcudnn" "-lcuda" "-lcudart" ];
 
     hardeningDisable = [ "all" ];
 
-    bazelFlags = [ "--config=opt" ]
-                 ++ lib.optional sse42Support "--copt=-msse4.2"
-                 ++ lib.optional avx2Support "--copt=-mavx2"
-                 ++ lib.optional fmaSupport "--copt=-mfma"
-                 ++ lib.optional cudaSupport "--config=cuda";
+    bazelFlags = [
+      # temporary fixes to make the build work with bazel 0.27
+      "--incompatible_no_support_tools_in_action_inputs=false"
+    ] ++ lib.optional sse42Support "--copt=-msse4.2"
+      ++ lib.optional avx2Support "--copt=-mavx2"
+      ++ lib.optional fmaSupport "--copt=-mfma";
 
     bazelTarget = "//tensorflow/tools/pip_package:build_pip_package";
 
@@ -101,44 +257,94 @@ let
         rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker,local_*,\@local_*}
       '';
 
-      sha256 = "1nc98aqrp14q7llypcwaa0kdn9xi7r0p1mnd3vmmn1m299py33ca";
+      # cudaSupport causes fetch of ncclArchive, resulting in different hashes
+      sha256 = if cudaSupport then
+        "0q1pmw7fzn6l554ap576r48m0zgwb7n1ljhyy1p36708z94scdh4"
+      else
+        "14n26h2r4w7wd5sddy4w0s51s2qcwf276n3hvv2505iysa8wqlc3";
     };
 
     buildAttrs = {
       preBuild = ''
         patchShebangs .
-        find -type f -name CROSSTOOL\* -exec sed -i \
-          -e 's,/usr/bin/ar,${binutils.bintools}/bin/ar,g' \
-          {} \;
+
+        # beautiful bash to iterate over files containing a string
+        # https://github.com/bazelbuild/bazel/issues/5915#issuecomment-505100422
+        # .. to make sure the output directory is covered
+        grep -lrZ '/usr/bin/ar\b' .. | while IFS="" read -r -d "" file; do
+            # patch /usr/bin/ar to the proper location
+            echo "File is $file"
+            sed -i \
+              -e 's,/usr/bin/ar\b,${binutils.bintools}/bin/ar,g' \
+              "$file"
+        done
+
+
+        # Tensorboard pulls in a bunch of dependencies, some of which may
+        # include security vulnerabilities. So we make it optional.
+        # https://github.com/tensorflow/tensorflow/issues/20280#issuecomment-400230560
+        sed -i '/tensorboard >=/d' tensorflow/tools/pip_package/setup.py
       '';
 
+      # Could alternatively use --src instead of --dst to output the sources
+      # instead of a whee. Generating a wheel makes it easier to unify handling
+      # of source and binary build though.
       installPhase = ''
-        sed -i 's,.*bdist_wheel.*,cp -rL . "$out"; exit 0,' bazel-bin/tensorflow/tools/pip_package/build_pip_package 
-        bazel-bin/tensorflow/tools/pip_package/build_pip_package $PWD/dist
+        # work around timestamp issues
+        # https://github.com/NixOS/nixpkgs/issues/270#issuecomment-467583872
+        export SOURCE_DATE_EPOCH=315532800
+
+        # bulid the wheel, then move it to $out (building directly to $out
+        # would actually put it into a *directory* called $out, but we want the
+        # file itself in $out)
+        bazel-bin/tensorflow/tools/pip_package/build_pip_package --dst $PWD/dist
+        mv dist/*.whl "$out"
       '';
     };
-
-    dontFixup = true;
   };
 
 in buildPythonPackage rec {
-  pname = "tensorflow";
-  inherit version;
+  inherit version pname;
 
-  src = pkg;
+  src = bazel-wheel;
 
-  installFlags = lib.optional (!withTensorboard) "--no-dependencies";
+  format = "wheel";
 
-  postPatch = lib.optionalString (pythonAtLeast "3.4") ''
-    sed -i '/enum34/d' setup.py
+  # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow
+  # and the propagated input tensorflow-tensorboard, which causes environment collisions.
+  # Another possibility would be to have tensorboard only in the buildInputs
+  # https://github.com/tensorflow/tensorflow/blob/v1.7.1/tensorflow/tools/pip_package/setup.py#L79
+  postInstall = ''
+    rm $out/bin/tensorboard
   '';
 
-  propagatedBuildInputs = [ numpy six protobuf absl-py ]
-                 ++ lib.optional (!isPy3k) mock
-                 ++ lib.optionals (pythonOlder "3.4") [ backports_weakref enum34 ]
-                 ++ lib.optional withTensorboard tensorflow-tensorboard;
+  # tensorflow/tools/pip_package/setup.py
+  propagatedBuildInputs = [
+    absl-py
+    astor
+    gast
+    google-pasta
+    keras-applications
+    keras-preprocessing
+    numpy
+    six
+    protobuf
+    tensorflow-estimator
+    termcolor
+    wrapt
+    grpcio
+  ] ++ lib.optionals (!isPy3k) [
+    mock
+    future # FIXME
+  ] ++ lib.optionals (pythonOlder "3.4") [
+    backports_weakref enum34
+  ] ++ lib.optionals withTensorboard [
+    tensorflow-tensorboard
+  ];
 
   # Actual tests are slow and impure.
+  # TODO try to run them anyway
+  # TODO better test (files in tensorflow/tools/ci_build/builds/*test)
   checkPhase = ''
     ${python.interpreter} -c "import tensorflow"
   '';
diff --git a/pkgs/development/python-modules/tensorflow/no-saved-proto.patch b/pkgs/development/python-modules/tensorflow/no-saved-proto.patch
new file mode 100644
index 000000000000..ead112f8ddd6
--- /dev/null
+++ b/pkgs/development/python-modules/tensorflow/no-saved-proto.patch
@@ -0,0 +1,14 @@
+diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
+index 8626ed0087..27deb34387 100644
+--- a/tensorflow/cc/saved_model/BUILD
++++ b/tensorflow/cc/saved_model/BUILD
+@@ -49,9 +49,6 @@ cc_library(
+         # tf_lib depending on the build platform.
+         "//tensorflow/core:lib",
+         "//tensorflow/core:protos_all_cc",
+-    ]) + if_mobile([
+-        # Mobile-friendly SavedModel proto. See go/portable-proto for more info.
+-        "//tensorflow/core:saved_model_portable_proto",
+     ]) + if_android([
+         "//tensorflow/core:android_tensorflow_lib",
+     ]) + if_ios([
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index be6a291b7c70..c28779bf4433 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -5703,11 +5703,13 @@ in {
   tensorflow =
     if stdenv.isDarwin
     then callPackage ../development/python-modules/tensorflow/bin.nix { }
-    else callPackage ../development/python-modules/tensorflow/bin.nix rec {
+    else callPackage ../development/python-modules/tensorflow rec {
       cudaSupport = pkgs.config.cudaSupport or false;
       inherit (pkgs.linuxPackages) nvidia_x11;
-      cudatoolkit = pkgs.cudatoolkit_10_0;
-      cudnn = pkgs.cudnn_cudatoolkit_10_0;
+      cudatoolkit = pkgs.cudatoolkit_10;
+      cudnn = pkgs.cudnn_cudatoolkit_10;
+      nccl = pkgs.nccl_cudatoolkit_10;
+      protobuf_cc = pkgs.protobuf; # not the python version
     };
 
   tensorflowWithoutCuda = self.tensorflow.override {
-- 
cgit 1.4.1


From 2e46ae0daa741031dd96d5bb089e390676878209 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Sun, 16 Jun 2019 14:58:23 +0200
Subject: libtensorflow: 1.9 -> 1.14.0

This also changes it to a from-source build.
---
 .../libraries/libtensorflow/default.nix            | 138 +++++++++++----------
 pkgs/top-level/all-packages.nix                    |   5 +-
 2 files changed, 76 insertions(+), 67 deletions(-)

diff --git a/pkgs/development/libraries/libtensorflow/default.nix b/pkgs/development/libraries/libtensorflow/default.nix
index 98096371e400..e4f6a868e9fd 100644
--- a/pkgs/development/libraries/libtensorflow/default.nix
+++ b/pkgs/development/libraries/libtensorflow/default.nix
@@ -1,80 +1,88 @@
-{ config, stdenv
-, fetchurl
-, patchelf
-, cudaSupport ? config.cudaSupport or false, symlinkJoin, cudatoolkit, cudnn, nvidia_x11
+{ stdenv
+, fetchurl, python3, which
+, patchelf, fetchFromGitHub, buildBazelPackage
+, cudatoolkit, cudnn, nccl, nvidia_x11, symlinkJoin
+, binutils, gcc, binutils-unwrapped, gcc-unwrapped
+, glibcLocales
+, features ? ["sse4.2" "avx" "avx2" "fma"]
+, arch ? "x86-64"
+, cudaSupport ? false
+, computeCapabilities ? [] # example: [ "7.5" ], see https://developer.nvidia.com/cuda-gpus
 }:
-with stdenv.lib;
 let
-  tfType = if cudaSupport then "gpu" else "cpu";
-  system =
-    if stdenv.isx86_64
-    then if      stdenv.isLinux  then "linux-x86_64"
-         else if stdenv.isDarwin then "darwin-x86_64" else unavailable
-    else unavailable;
-  unavailable = throw "libtensorflow is not available for this platform!";
+  inherit (stdenv) lib;
   cudatoolkit_joined = symlinkJoin {
     name = "unsplit_cudatoolkit";
     paths = [ cudatoolkit.out
               cudatoolkit.lib ];};
-  rpath = makeLibraryPath ([stdenv.cc.libc stdenv.cc.cc.lib] ++
-            optionals cudaSupport [ cudatoolkit_joined cudnn nvidia_x11 ]);
-  patchLibs =
-    if stdenv.isDarwin
-    then ''
-      install_name_tool -id $out/lib/libtensorflow.so $out/lib/libtensorflow.so
-      install_name_tool -id $out/lib/libtensorflow_framework.so $out/lib/libtensorflow_framework.so
-    ''
-    else ''
-      ${patchelf}/bin/patchelf --set-rpath "${rpath}:$out/lib" $out/lib/libtensorflow.so
-      ${patchelf}/bin/patchelf --set-rpath "${rpath}" $out/lib/libtensorflow_framework.so
-    '';
-
-in stdenv.mkDerivation rec {
-  pname = "libtensorflow";
-  version = "1.9.0";
+  gcc_joined = symlinkJoin {
+    name = "gcc-joined";
+    paths = [ binutils gcc binutils-unwrapped gcc-unwrapped ];
+  };
+in
+buildBazelPackage rec {
+  pname = "tensorflow";
+  version = "1.14.0";
   name = "${pname}-${version}";
-  src = fetchurl {
-    url = "https://storage.googleapis.com/tensorflow/${pname}/${pname}-${tfType}-${system}-${version}.tar.gz";
-    sha256 =
-      if system == "linux-x86_64" then
-        if cudaSupport
-        then "1q3mh06x344im25z7r3vgrfksfdsi8fh8ldn6y2mf86h4d11yxc3"
-        else "0l9ps115ng5ffzdwphlqmj3jhidps2v5afppdzrbpzmy41xz0z21"
-      else if system == "darwin-x86_64" then
-        if cudaSupport
-        then unavailable
-        else "1qj0v1706w6mczycdsh38h2glyv5d25v62kdn98wxd5rw8f9v657"
-      else unavailable;
+  bazelFlags = [ "--incompatible_no_support_tools_in_action_inputs=false" ];
+  bazelTarget = "//tensorflow/tools/lib_package:libtensorflow";
+  fetchAttrs = {
+    sha256 = if cudaSupport then
+      "127xxwy3a2h1qsv2sqfhrh65g69hlb1q003vyyg7yjfqgfah9p2z"
+      else "1di1pnknr1hxdpn75lxf9c6dvb5kgllmgb9r9rgh5c2g9iil17zy";
+  };
+  CC_OPT_FLAGS = "-march=${arch} " + stdenv.lib.concatMapStringsSep " " (f: "-m"+f) features;
+  NIX_CFLAGS_COMPILE = CC_OPT_FLAGS;
+  TF_NEED_CUDA = if cudaSupport then "1" else "0";
+  TF_IGNORE_MAX_BAZEL_VERSION = "1";
+  TF_CUDA_COMPUTE_CAPABILITIES = stdenv.lib.concatStringsSep "," computeCapabilities;
+  nativeBuildInputs = [ glibcLocales python3 which ]
+    ++ (lib.optionals cudaSupport [
+    cudatoolkit_joined cudnn nvidia_x11
+  ]);
+  TF_CUDA_PATHS = lib.optionalString cudaSupport "${cudatoolkit_joined},${cudnn}";
+  src = fetchFromGitHub {
+    repo = "tensorflow";
+    owner = "tensorflow";
+    rev = "v${version}";
+    sha256 = "06jvwlsm14b8rqwd8q8796r0vmn0wk64s4ps2zg0sapkmp9vvcmi";
   };
-
-  # Patch library to use our libc, libstdc++ and others
-  buildCommand = ''
-    . $stdenv/setup
-    mkdir -pv $out
-    tar -C $out -xzf $src
-    chmod +w $out/lib/libtensorflow.so
-    chmod +w $out/lib/libtensorflow_framework.so
-    ${patchLibs}
-    chmod -w $out/lib/libtensorflow.so
-    chmod -w $out/lib/libtensorflow_framework.so
-
-    # Write pkgconfig file.
-    mkdir $out/lib/pkgconfig
-    cat > $out/lib/pkgconfig/tensorflow.pc << EOF
-    Name: TensorFlow
-    Version: ${version}
-    Description: Library for computation using data flow graphs for scalable machine learning
-    Requires:
-    Libs: -L$out/lib -ltensorflow
-    Cflags: -I$out/include/tensorflow
-    EOF
+  prePatch = ''
+    # doesn't work:
+    sed -i '/saved_model_portable_proto/d' tensorflow/cc/saved_model/BUILD
+    # calls ldconfig -p:
+    sed -i 's/+ _get_ld_config_paths()//' third_party/gpus/find_cuda_config.py
+    patchShebangs ./configure
+    patchShebangs tensorflow/tools/lib_package/concat_licenses.sh
+    patchShebangs third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl
   '';
-
-  meta = {
+  GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${gcc_joined}/bin";
+  configurePhase = ''
+    runHook preConfigure
+    ./configure
+    runHook postConfigure
+  '';
+  buildAttrs = {
+    installPhase = ''
+      mkdir -p $out
+      tar -xf bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz -C $out
+      # Write pkgconfig file.
+      mkdir $out/lib/pkgconfig
+      cat > $out/lib/pkgconfig/tensorflow.pc << EOF
+      Name: TensorFlow
+      Version: ${version}
+      Description: Library for computation using data flow graphs for scalable machine learning
+      Requires:
+      Libs: -L$out/lib -ltensorflow
+      Cflags: -I$out/include/tensorflow
+      EOF
+    '';
+  };
+  meta = with lib; {
     description = "C API for TensorFlow";
     homepage = https://www.tensorflow.org/versions/master/install/install_c;
     license = licenses.asl20;
     platforms = with platforms; linux ++ darwin;
-    maintainers = [maintainers.basvandijk];
+    maintainers = with maintainers; [ basvandijk yorickvp ];
   };
 }
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 0d35b7c9f390..a9342085d345 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -1095,8 +1095,9 @@ in
 
   libtensorflow = callPackage ../development/libraries/libtensorflow {
     inherit (linuxPackages) nvidia_x11;
-    cudatoolkit = cudatoolkit_9_0;
-    cudnn = cudnn_cudatoolkit_9_0;
+    cudatoolkit = cudatoolkit_10_0;
+    cudnn = cudnn_cudatoolkit_10_0;
+    nccl = nccl_cudatoolkit_10;
   };
 
   behdad-fonts = callPackage ../data/fonts/behdad-fonts { };
-- 
cgit 1.4.1


From 170dd558dffd2bb134e3001a93631fc71fe13f3c Mon Sep 17 00:00:00 2001
From: Nikolay Amiantov <ab@fmap.me>
Date: Sat, 13 Jul 2019 22:37:01 +0300
Subject: python.pkgs.tensorflow: cleanup binary build

---
 pkgs/development/python-modules/tensorflow/bin.nix | 25 +++++++------
 .../python-modules/tensorflow/binary-hashes.nix    | 43 ++++++++++++++++++++++
 .../python-modules/tensorflow/prefetcher.sh        |  5 ++-
 .../python-modules/tensorflow/tf1.14.0-hashes.nix  | 42 ---------------------
 4 files changed, 61 insertions(+), 54 deletions(-)
 create mode 100644 pkgs/development/python-modules/tensorflow/binary-hashes.nix
 mode change 100644 => 100755 pkgs/development/python-modules/tensorflow/prefetcher.sh
 delete mode 100644 pkgs/development/python-modules/tensorflow/tf1.14.0-hashes.nix

diff --git a/pkgs/development/python-modules/tensorflow/bin.nix b/pkgs/development/python-modules/tensorflow/bin.nix
index 817ba09dc9d2..a562ba8f2fad 100644
--- a/pkgs/development/python-modules/tensorflow/bin.nix
+++ b/pkgs/development/python-modules/tensorflow/bin.nix
@@ -35,15 +35,19 @@
 assert cudaSupport -> cudatoolkit != null
                    && cudnn != null
                    && nvidia_x11 != null;
+
+# unsupported combination
+assert ! (stdenv.isDarwin && cudaSupport);
+
 let
-  cudatoolkit_joined = symlinkJoin {
-    name = "unsplit_cudatoolkit";
-    paths = [ cudatoolkit.out
-              cudatoolkit.lib ];};
+  packages = import ./binary-hashes.nix;
+
+  variant = if cudaSupport then "-gpu" else "";
+  pname = "tensorflow${variant}";
 
 in buildPythonPackage rec {
-  pname = "tensorflow";
-  version = "1.14.0";
+  inherit pname;
+  inherit (packages) version;
   format = "wheel";
 
   src = let
@@ -52,8 +56,7 @@ in buildPythonPackage rec {
     platform = if stdenv.isDarwin then "mac" else "linux";
     unit = if cudaSupport then "gpu" else "cpu";
     key = "${platform}_py_${pyver}_${unit}";
-    dls = import (./. + "/tf${version}-hashes.nix");
-  in fetchurl dls.${key};
+  in fetchurl packages.${key};
 
   propagatedBuildInputs = [
     protobuf
@@ -86,9 +89,9 @@ in buildPythonPackage rec {
   # patchelf --shrink-rpath will remove the cuda libraries.
   postFixup = let
     rpath = stdenv.lib.makeLibraryPath
-      ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit_joined cudnn nvidia_x11 ]);
+      ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]);
   in
-  lib.optionalString (stdenv.isLinux) ''
+  lib.optionalString stdenv.isLinux ''
     rrPath="$out/${python.sitePackages}/tensorflow/:$out/${python.sitePackages}/tensorflow/contrib/tensor_forest/:${rpath}"
     internalLibPath="$out/${python.sitePackages}/tensorflow/python/_pywrap_tensorflow_internal.so"
     find $out \( -name '*.so' -or -name '*.so.*' \) -exec patchelf --set-rpath "$rrPath" {} \;
@@ -100,7 +103,7 @@ in buildPythonPackage rec {
     homepage = http://tensorflow.org;
     license = licenses.asl20;
     maintainers = with maintainers; [ jyp abbradar ];
-    platforms = with platforms; linux ++ lib.optionals (!cudaSupport) darwin;
+    platforms = [ "x86_64-linux" "x86_64-darwin" ];
     # Python 2.7 build uses different string encoding.
     # See https://github.com/NixOS/nixpkgs/pull/37044#issuecomment-373452253
     broken = stdenv.isDarwin && !isPy3k;
diff --git a/pkgs/development/python-modules/tensorflow/binary-hashes.nix b/pkgs/development/python-modules/tensorflow/binary-hashes.nix
new file mode 100644
index 000000000000..fa4809dc3f19
--- /dev/null
+++ b/pkgs/development/python-modules/tensorflow/binary-hashes.nix
@@ -0,0 +1,43 @@
+{
+version = "1.14.0";
+linux_py_27_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp27-none-linux_x86_64.whl";
+  sha256 = "0yywdrfk97dh1bxhibspg0raz70fx9lcczj6xlimqy4xb60clx7k";
+};
+linux_py_35_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp35-cp35m-linux_x86_64.whl";
+  sha256 = "1xvyb6xcrjhlwvrmrhn5vs9xy7g98smqmpv4i3hhpry4qyasphhj";
+};
+linux_py_36_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl";
+  sha256 = "1psd9vyxz9f39dwj77nvrg373sxv3p5vdp9fnz81dpsm0b0mwl44";
+};
+linux_py_37_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp37-cp37m-linux_x86_64.whl";
+  sha256 = "0bg2sb1n2ag27r7ww695kg5hb0mjrw4kc5893krmixx2j71860c5";
+};
+linux_py_27_gpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp27-none-linux_x86_64.whl";
+  sha256 = "0y1x91gayg6pjddgl8ndcm63wfzhyv4s5khgl7ffzsgni1ivaqw5";
+};
+linux_py_35_gpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp35-cp35m-linux_x86_64.whl";
+  sha256 = "03piggpbz1jx8m2b95spq3jrdff4w6xx63ji07am7hyw2nsgx3mx";
+};
+linux_py_36_gpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp36-cp36m-linux_x86_64.whl";
+  sha256 = "0ypkp8cfhharsyyikb1qgf44cfm6284km9xswzvzymjzz75vg3gd";
+};
+linux_py_37_gpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp37-cp37m-linux_x86_64.whl";
+  sha256 = "0virp8nn2ysx4855hq29kas6fm6b3dsiybwzdxy9nnb9n2d8qlm2";
+};
+mac_py_2_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py2-none-any.whl";
+  sha256 = "14f86k3pgq7z6i4s4im55zpp38f0drnm7xlclavsgcc0nxnj3z26";
+};
+mac_py_3_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py3-none-any.whl";
+  sha256 = "0f3swpcjfgqhj6h5wnx8snc0xjkx4hnkqx83fmlrwpncs8c131d3";
+};
+}
diff --git a/pkgs/development/python-modules/tensorflow/prefetcher.sh b/pkgs/development/python-modules/tensorflow/prefetcher.sh
old mode 100644
new mode 100755
index 3059340d127d..d590fb0f1732
--- a/pkgs/development/python-modules/tensorflow/prefetcher.sh
+++ b/pkgs/development/python-modules/tensorflow/prefetcher.sh
@@ -1,7 +1,10 @@
+#!/usr/bin/env bash
+
 version=1.14.0
-hashfile=tf${version}-hashes.nix
+hashfile=binary-hashes.nix
 rm -f $hashfile
 echo "{" >> $hashfile
+echo "version = \"$version\";" >> $hashfile
 for sys in "linux" "mac"; do
     for tfpref in "cpu/tensorflow" "gpu/tensorflow_gpu"; do
         for pykind in "py2-none-any" "py3-none-any" "cp27-none-linux_x86_64" "cp35-cp35m-linux_x86_64" "cp36-cp36m-linux_x86_64" "cp37-cp37m-linux_x86_64"; do
diff --git a/pkgs/development/python-modules/tensorflow/tf1.14.0-hashes.nix b/pkgs/development/python-modules/tensorflow/tf1.14.0-hashes.nix
deleted file mode 100644
index f6b367aea383..000000000000
--- a/pkgs/development/python-modules/tensorflow/tf1.14.0-hashes.nix
+++ /dev/null
@@ -1,42 +0,0 @@
-{
-linux_py_27_cpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp27-none-linux_x86_64.whl";
-  sha256 = "0yywdrfk97dh1bxhibspg0raz70fx9lcczj6xlimqy4xb60clx7k";
-};
-linux_py_35_cpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp35-cp35m-linux_x86_64.whl";
-  sha256 = "1xvyb6xcrjhlwvrmrhn5vs9xy7g98smqmpv4i3hhpry4qyasphhj";
-};
-linux_py_36_cpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl";
-  sha256 = "1psd9vyxz9f39dwj77nvrg373sxv3p5vdp9fnz81dpsm0b0mwl44";
-};
-linux_py_37_cpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp37-cp37m-linux_x86_64.whl";
-  sha256 = "0bg2sb1n2ag27r7ww695kg5hb0mjrw4kc5893krmixx2j71860c5";
-};
-linux_py_27_gpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp27-none-linux_x86_64.whl";
-  sha256 = "0y1x91gayg6pjddgl8ndcm63wfzhyv4s5khgl7ffzsgni1ivaqw5";
-};
-linux_py_35_gpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp35-cp35m-linux_x86_64.whl";
-  sha256 = "03piggpbz1jx8m2b95spq3jrdff4w6xx63ji07am7hyw2nsgx3mx";
-};
-linux_py_36_gpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp36-cp36m-linux_x86_64.whl";
-  sha256 = "0ypkp8cfhharsyyikb1qgf44cfm6284km9xswzvzymjzz75vg3gd";
-};
-linux_py_37_gpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp37-cp37m-linux_x86_64.whl";
-  sha256 = "0virp8nn2ysx4855hq29kas6fm6b3dsiybwzdxy9nnb9n2d8qlm2";
-};
-mac_py_2_cpu = {
-  url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py2-none-any.whl";
-  sha256 = "14f86k3pgq7z6i4s4im55zpp38f0drnm7xlclavsgcc0nxnj3z26";
-};
-mac_py_3_cpu = {
-  url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py3-none-any.whl";
-  sha256 = "0f3swpcjfgqhj6h5wnx8snc0xjkx4hnkqx83fmlrwpncs8c131d3";
-};
-}
-- 
cgit 1.4.1


From 0a1bf4734fb93ebc7d2240a9d98babdacfcf6100 Mon Sep 17 00:00:00 2001
From: Nikolay Amiantov <ab@fmap.me>
Date: Sat, 13 Jul 2019 22:38:41 +0300
Subject: libtensorflow: add binary build and add automatic generation

---
 .../libraries/libtensorflow/default.nix            | 88 ----------------------
 .../libraries/science/math/tensorflow/bin.nix      | 72 ++++++++++++++++++
 .../science/math/tensorflow/binary-hashes.nix      | 15 ++++
 .../science/math/tensorflow/prefetcher.sh          | 24 ++++++
 pkgs/top-level/all-packages.nix                    |  4 +-
 5 files changed, 113 insertions(+), 90 deletions(-)
 delete mode 100644 pkgs/development/libraries/libtensorflow/default.nix
 create mode 100644 pkgs/development/libraries/science/math/tensorflow/bin.nix
 create mode 100644 pkgs/development/libraries/science/math/tensorflow/binary-hashes.nix
 create mode 100755 pkgs/development/libraries/science/math/tensorflow/prefetcher.sh

diff --git a/pkgs/development/libraries/libtensorflow/default.nix b/pkgs/development/libraries/libtensorflow/default.nix
deleted file mode 100644
index e4f6a868e9fd..000000000000
--- a/pkgs/development/libraries/libtensorflow/default.nix
+++ /dev/null
@@ -1,88 +0,0 @@
-{ stdenv
-, fetchurl, python3, which
-, patchelf, fetchFromGitHub, buildBazelPackage
-, cudatoolkit, cudnn, nccl, nvidia_x11, symlinkJoin
-, binutils, gcc, binutils-unwrapped, gcc-unwrapped
-, glibcLocales
-, features ? ["sse4.2" "avx" "avx2" "fma"]
-, arch ? "x86-64"
-, cudaSupport ? false
-, computeCapabilities ? [] # example: [ "7.5" ], see https://developer.nvidia.com/cuda-gpus
-}:
-let
-  inherit (stdenv) lib;
-  cudatoolkit_joined = symlinkJoin {
-    name = "unsplit_cudatoolkit";
-    paths = [ cudatoolkit.out
-              cudatoolkit.lib ];};
-  gcc_joined = symlinkJoin {
-    name = "gcc-joined";
-    paths = [ binutils gcc binutils-unwrapped gcc-unwrapped ];
-  };
-in
-buildBazelPackage rec {
-  pname = "tensorflow";
-  version = "1.14.0";
-  name = "${pname}-${version}";
-  bazelFlags = [ "--incompatible_no_support_tools_in_action_inputs=false" ];
-  bazelTarget = "//tensorflow/tools/lib_package:libtensorflow";
-  fetchAttrs = {
-    sha256 = if cudaSupport then
-      "127xxwy3a2h1qsv2sqfhrh65g69hlb1q003vyyg7yjfqgfah9p2z"
-      else "1di1pnknr1hxdpn75lxf9c6dvb5kgllmgb9r9rgh5c2g9iil17zy";
-  };
-  CC_OPT_FLAGS = "-march=${arch} " + stdenv.lib.concatMapStringsSep " " (f: "-m"+f) features;
-  NIX_CFLAGS_COMPILE = CC_OPT_FLAGS;
-  TF_NEED_CUDA = if cudaSupport then "1" else "0";
-  TF_IGNORE_MAX_BAZEL_VERSION = "1";
-  TF_CUDA_COMPUTE_CAPABILITIES = stdenv.lib.concatStringsSep "," computeCapabilities;
-  nativeBuildInputs = [ glibcLocales python3 which ]
-    ++ (lib.optionals cudaSupport [
-    cudatoolkit_joined cudnn nvidia_x11
-  ]);
-  TF_CUDA_PATHS = lib.optionalString cudaSupport "${cudatoolkit_joined},${cudnn}";
-  src = fetchFromGitHub {
-    repo = "tensorflow";
-    owner = "tensorflow";
-    rev = "v${version}";
-    sha256 = "06jvwlsm14b8rqwd8q8796r0vmn0wk64s4ps2zg0sapkmp9vvcmi";
-  };
-  prePatch = ''
-    # doesn't work:
-    sed -i '/saved_model_portable_proto/d' tensorflow/cc/saved_model/BUILD
-    # calls ldconfig -p:
-    sed -i 's/+ _get_ld_config_paths()//' third_party/gpus/find_cuda_config.py
-    patchShebangs ./configure
-    patchShebangs tensorflow/tools/lib_package/concat_licenses.sh
-    patchShebangs third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl
-  '';
-  GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${gcc_joined}/bin";
-  configurePhase = ''
-    runHook preConfigure
-    ./configure
-    runHook postConfigure
-  '';
-  buildAttrs = {
-    installPhase = ''
-      mkdir -p $out
-      tar -xf bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz -C $out
-      # Write pkgconfig file.
-      mkdir $out/lib/pkgconfig
-      cat > $out/lib/pkgconfig/tensorflow.pc << EOF
-      Name: TensorFlow
-      Version: ${version}
-      Description: Library for computation using data flow graphs for scalable machine learning
-      Requires:
-      Libs: -L$out/lib -ltensorflow
-      Cflags: -I$out/include/tensorflow
-      EOF
-    '';
-  };
-  meta = with lib; {
-    description = "C API for TensorFlow";
-    homepage = https://www.tensorflow.org/versions/master/install/install_c;
-    license = licenses.asl20;
-    platforms = with platforms; linux ++ darwin;
-    maintainers = with maintainers; [ basvandijk yorickvp ];
-  };
-}
diff --git a/pkgs/development/libraries/science/math/tensorflow/bin.nix b/pkgs/development/libraries/science/math/tensorflow/bin.nix
new file mode 100644
index 000000000000..3db7a6f1d1e2
--- /dev/null
+++ b/pkgs/development/libraries/science/math/tensorflow/bin.nix
@@ -0,0 +1,72 @@
+{ stdenv
+, fetchurl
+, patchelf
+, cudaSupport ? false, symlinkJoin, cudatoolkit, cudnn, nvidia_x11
+}:
+
+with stdenv.lib;
+let
+  unavailable = throw "libtensorflow is not available for this platform!";
+
+  tfType = if cudaSupport then "gpu" else "cpu";
+
+  system = 
+    if      stdenv.isLinux  then "linux"
+    else if stdenv.isDarwin then "darwin"
+    else unavailable;
+
+  platform =
+    if stdenv.isx86_64 then "x86_64"
+    else unavailable;
+
+  rpath = makeLibraryPath ([stdenv.cc.libc stdenv.cc.cc.lib] ++
+            optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]);
+
+  packages = import ./binary-hashes.nix;
+  packageName = "${tfType}-${system}-${platform}";
+  url = packages.${packageName} or unavailable;
+
+  patchLibs =
+    if stdenv.isDarwin
+    then ''
+      install_name_tool -id $out/lib/libtensorflow.dylib $out/lib/libtensorflow.dylib
+      install_name_tool -id $out/lib/libtensorflow_framework.dylib $out/lib/libtensorflow_framework.dylib
+    ''
+    else ''
+      patchelf --set-rpath "${rpath}:$out/lib" $out/lib/libtensorflow.so
+      patchelf --set-rpath "${rpath}" $out/lib/libtensorflow_framework.so
+    '';
+
+in stdenv.mkDerivation rec {
+  pname = "libtensorflow";
+  inherit (packages) version;
+
+  src = fetchurl url;
+
+  # Patch library to use our libc, libstdc++ and others
+  buildCommand = ''
+    mkdir -pv $out
+    tar -C $out -xzf $src
+    chmod -R +w $out
+    ${patchLibs}
+
+    # Write pkgconfig file.
+    mkdir $out/lib/pkgconfig
+    cat > $out/lib/pkgconfig/tensorflow.pc << EOF
+    Name: TensorFlow
+    Version: ${version}
+    Description: Library for computation using data flow graphs for scalable machine learning
+    Requires:
+    Libs: -L$out/lib -ltensorflow
+    Cflags: -I$out/include/tensorflow
+    EOF
+  '';
+
+  meta = {
+    description = "C API for TensorFlow";
+    homepage = https://www.tensorflow.org/install/lang_c;
+    license = licenses.asl20;
+    platforms = [ "x86_64-linux" "x86_64-darwin" ];
+    maintainers = with maintainers; [ basvandijk ];
+  };
+}
diff --git a/pkgs/development/libraries/science/math/tensorflow/binary-hashes.nix b/pkgs/development/libraries/science/math/tensorflow/binary-hashes.nix
new file mode 100644
index 000000000000..892dfa2a6097
--- /dev/null
+++ b/pkgs/development/libraries/science/math/tensorflow/binary-hashes.nix
@@ -0,0 +1,15 @@
+{
+version = "1.14.0";
+"cpu-linux-x86_64" = {
+  url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz";
+  sha256 = "04bi3ijq4sbb8c5vk964zlv0j9mrjnzzxd9q9knq3h273nc1a36k";
+};
+"gpu-linux-x86_64" = {
+  url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-1.14.0.tar.gz";
+  sha256 = "1ffnpyj9jjgwxpjfiyjvq4dm3n6nwiksim5jld9zw7fdswh215x6";
+};
+"cpu-darwin-x86_64" = {
+  url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-darwin-x86_64-1.14.0.tar.gz";
+  sha256 = "0zsd5ils1a17j6jzh0c7q1z56fw46gkzybbnms7h2rgg8al0rh92";
+};
+}
diff --git a/pkgs/development/libraries/science/math/tensorflow/prefetcher.sh b/pkgs/development/libraries/science/math/tensorflow/prefetcher.sh
new file mode 100755
index 000000000000..515f25df4c1f
--- /dev/null
+++ b/pkgs/development/libraries/science/math/tensorflow/prefetcher.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+version=1.14.0
+hashfile=binary-hashes.nix
+rm -f $hashfile
+echo "{" >> $hashfile
+echo "version = \"$version\";" >> $hashfile
+for sys in "linux" "darwin"; do
+    for tfpref in "cpu" "gpu"; do
+        for platform in "x86_64"; do
+            if [ $sys = "darwin" ] && [ $tfpref = "gpu" ]; then
+               continue
+            fi
+            url=https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-$tfpref-$sys-$platform-$version.tar.gz
+            hash=$(nix-prefetch-url $url)
+            echo "\"${tfpref}-${sys}-${platform}\" = {" >> $hashfile
+            echo "  url = \"$url\";" >> $hashfile
+            echo "  sha256 = \"$hash\";" >> $hashfile
+            echo "};" >> $hashfile
+        done
+    done
+done
+echo "}" >> $hashfile
+
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index a9342085d345..a16369a1e072 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -1093,11 +1093,11 @@ in
 
   tensor = libsForQt5.callPackage ../applications/networking/instant-messengers/tensor { };
 
-  libtensorflow = callPackage ../development/libraries/libtensorflow {
+  libtensorflow = callPackage ../development/libraries/science/math/tensorflow/bin.nix {
+    cudaSupport = pkgs.config.cudaSupport or false;
     inherit (linuxPackages) nvidia_x11;
     cudatoolkit = cudatoolkit_10_0;
     cudnn = cudnn_cudatoolkit_10_0;
-    nccl = nccl_cudatoolkit_10;
   };
 
   behdad-fonts = callPackage ../data/fonts/behdad-fonts { };
-- 
cgit 1.4.1


From 019c13616b512d0f8b3bd3e5e17356401a8e4457 Mon Sep 17 00:00:00 2001
From: Nikolay Amiantov <ab@fmap.me>
Date: Sat, 13 Jul 2019 22:42:50 +0300
Subject: tensorflow: cleanup and merge with libtensorflow

This merges work done by yorickvP and timokau in #63208 and #63616 respectively.
Now the derivation builds both libtensorflow and the Python package and puts them into
different outputs.

Quite a bit of improvements were done on the top, including:

* Use official tag revision as source, not a branch;
* Use all system libraries possible (before only one was actually used);
* Move various environment variables to the derivation itself from hooks;
* Use source Python build instead of wheel build to ensure fixup hooks do their important jobs on libraries;
* And more that I forgot!
---
 .../python-modules/tensorflow/default.nix          | 269 ++++++++++-----------
 .../python-modules/tensorflow/system-jsoncpp.patch |  21 ++
 pkgs/top-level/all-packages.nix                    |  15 +-
 pkgs/top-level/python-packages.nix                 |   8 +-
 4 files changed, 170 insertions(+), 143 deletions(-)
 create mode 100644 pkgs/development/python-modules/tensorflow/system-jsoncpp.patch

diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix
index 98e651a5d95c..d2d8d50fda46 100644
--- a/pkgs/development/python-modules/tensorflow/default.nix
+++ b/pkgs/development/python-modules/tensorflow/default.nix
@@ -1,41 +1,15 @@
-{ stdenv, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin
-, git
-, buildPythonPackage, isPy3k, pythonOlder, pythonAtLeast
-, which, binutils, glibcLocales
-, python, jemalloc, openmpi
+{ stdenv, pkgs, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin
+# Python deps
+, buildPythonPackage, isPy3k, pythonOlder, pythonAtLeast, python
+# Python libraries
 , numpy, tensorflow-tensorboard, backports_weakref, mock, enum34, absl-py
-, future
-, keras-preprocessing
-, keras-applications
-, astor
-, gast
-, google-pasta
-, termcolor
-, cython
-, flatbuffers
-, giflib
-, libjpeg
-, grpc
-, grpcio
-, hwloc
-, icu
-, jsoncpp
-, lmdb
-, nasm
-, sqlite
-, pcre
-, libpng
-, six
-, snappy
-, swig
-, wrapt
-, zlib
-, protobuf
-, protobuf_cc
-, curl
-, tensorflow-estimator
-, setuptools
-, wheel
+, future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta
+, termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator
+# Common deps
+, git, swig, which, binutils, glibcLocales, cython
+# Common libraries
+, jemalloc, openmpi, astor, gast, grpc, sqlite, openssl, jsoncpp, re2
+, curl, snappy, flatbuffers, icu, double-conversion, libpng, libjpeg, giflib
 , cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null, nccl ? null
 # XLA without CUDA is broken
 , xlaSupport ? cudaSupport
@@ -57,48 +31,51 @@ let
   withTensorboard = pythonOlder "3.6";
 
   cudatoolkit_joined = symlinkJoin {
-    name = "${cudatoolkit.name}-unsplit";
+    name = "${cudatoolkit.name}-merged";
     paths = [ cudatoolkit.out cudatoolkit.lib ];
   };
 
-  bazel_cuda_toolkit = symlinkJoin {
-    name = "bazel-gcc-toolkit";
+  cudatoolkit_cc_joined = symlinkJoin {
+    name = "${cudatoolkit.cc.name}-merged";
     paths = [
       cudatoolkit.cc
-      cudatoolkit.out
-      cudatoolkit.lib
       binutils.bintools # for ar, dwp, nm, objcopy, objdump, strip
     ];
   };
 
+  # Needed for _some_ system libraries, grep INCLUDEDIR.
+  includes_joined = symlinkJoin {
+    name = "tensorflow-deps-merged";
+    paths = [
+      pkgs.protobuf
+      jsoncpp
+    ];
+  };
+
   tfFeature = x: if x then "1" else "0";
 
-  version = "1.14";
-  variant = if cudaSupport then "_gpu" else "";
+  version = "1.14.0";
+  variant = if cudaSupport then "-gpu" else "";
   pname = "tensorflow${variant}";
 
-  bazel-wheel = buildBazelPackage rec {
-    # indicate which configuration of the wheel is being built
-    name = let
-      python_tag = with lib.versions; "cp${major python.version}${minor python.version}"; # cp37
-      arch_platform = lib.splitString "-" stdenv.hostPlatform.system;
-      platform = lib.last arch_platform; # "linux"
-      arch = lib.head arch_platform; # "x86_64"
-    in
-      # https://www.python.org/dev/peps/pep-0427/#file-name-convention
-      # tensorflow_gpu-1.14.0-cp27-none-linux_x86_64.whl
-      "${pname}-${version}-${python_tag}-none-${platform}_${arch}.whl";
+  # TODO: remove after there's support for setupPyDistFlags
+  setuppy = ../../../development/interpreters/python/run_setup.py;
+
+  bazel-build = buildBazelPackage rec {
+    name = "${pname}-${version}";
 
     src = fetchFromGitHub {
       owner = "tensorflow";
       repo = "tensorflow";
-      rev = "r${version}";
-      sha256 = "071rndqh3xy4y5idlykqv656lggp25vy4qx1divz4id3im1zlr13";
+      rev = "v${version}";
+      sha256 = "06jvwlsm14b8rqwd8q8796r0vmn0wk64s4ps2zg0sapkmp9vvcmi";
     };
 
     patches = [
       # Work around https://github.com/tensorflow/tensorflow/issues/24752
       ./no-saved-proto.patch
+      # Fixes for NixOS jsoncpp
+      ./system-jsoncpp.patch
 
       # https://github.com/tensorflow/tensorflow/pull/29673
       (fetchpatch {
@@ -118,7 +95,9 @@ let
     # On update, it can be useful to steal the changes from gentoo
     # https://gitweb.gentoo.org/repo/gentoo.git/tree/sci-libs/tensorflow
 
-    nativeBuildInputs = [ swig which ];
+    nativeBuildInputs = [
+      swig which cython
+    ];
 
     buildInputs = [
       python
@@ -130,9 +109,30 @@ let
       # python deps needed during wheel build time
       numpy
       keras-preprocessing
+      protobuf
+      wrapt
+      gast
+      astor
+      absl-py
+      termcolor
+      keras-applications
 
       # libs taken from system through the TF_SYS_LIBS mechanism
-      absl-py
+      grpc
+      sqlite
+      openssl
+      jsoncpp
+      pkgs.protobuf
+      curl
+      snappy
+      flatbuffers
+      icu
+      double-conversion
+      libpng
+      libjpeg
+      giflib
+      re2
+      pkgs.lmdb
 
       # for building the wheel
       setuptools
@@ -146,22 +146,26 @@ let
       nvidia_x11
     ];
 
+
+    # arbitrarily set to the current latest bazel version, overly careful
+    TF_IGNORE_MAX_BAZEL_VERSION = true;
+
     # Take as many libraries from the system as possible. Keep in sync with
     # list of valid syslibs in
-    # https://github.com/perfinion/tensorflow/blob/master/third_party/systemlibs/syslibs_configure.bzl
-    SYSLIBS= [
+    # https://github.com/tensorflow/tensorflow/blob/master/third_party/systemlibs/syslibs_configure.bzl
+    TF_SYSTEM_LIBS = lib.concatStringsSep "," [
       "absl_py"
       "astor_archive"
       "boringssl"
-      "com_github_googleapis_googleapis"
-      "com_github_googlecloudplatform_google_cloud_cpp"
+      # Not packaged in nixpkgs
+      # "com_github_googleapis_googleapis"
+      # "com_github_googlecloudplatform_google_cloud_cpp"
       "com_google_protobuf"
       "com_google_protobuf_cc"
       "com_googlesource_code_re2"
       "curl"
       "cython"
       "double_conversion"
-      "enum34_archive"
       "flatbuffers"
       "gast_archive"
       "gif_archive"
@@ -174,7 +178,7 @@ let
       "lmdb"
       "nasm"
       # "nsync" # not packaged in nixpkgs
-      "sqlite"
+      "org_sqlite"
       "pasta"
       "pcre"
       "png_archive"
@@ -187,6 +191,34 @@ let
       "zlib_archive"
     ];
 
+    INCLUDEDIR = "${includes_joined}/include";
+
+    PYTHON_BIN_PATH = python.interpreter;
+ 
+    TF_NEED_GCP = true;
+    TF_NEED_HDFS = true;
+    TF_ENABLE_XLA = tfFeature xlaSupport;
+
+    CC_OPT_FLAGS = " ";
+
+    # https://github.com/tensorflow/tensorflow/issues/14454
+    TF_NEED_MPI = tfFeature cudaSupport;
+
+    TF_NEED_CUDA = tfFeature cudaSupport;
+    TF_CUDA_PATHS = lib.optionalString cudaSupport "${cudatoolkit_joined},${cudnn},${nccl}";
+    GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin";
+    TF_CUDA_COMPUTE_CAPABILITIES = lib.concatStringsSep "," cudaCapabilities;
+
+    postPatch = ''
+      # https://github.com/tensorflow/tensorflow/issues/20919
+      sed -i '/androidndk/d' tensorflow/lite/kernels/internal/BUILD
+
+      # Tensorboard pulls in a bunch of dependencies, some of which may
+      # include security vulnerabilities. So we make it optional.
+      # https://github.com/tensorflow/tensorflow/issues/20280#issuecomment-400230560
+      sed -i '/tensorboard >=/d' tensorflow/tools/pip_package/setup.py
+    '';
+
     preConfigure = ''
       patchShebangs configure
 
@@ -196,52 +228,21 @@ let
       chmod +x dummy-ldconfig/ldconfig
       export PATH="$PWD/dummy-ldconfig:$PATH"
 
-      # arbitrarily set to the current latest bazel version, overly careful
-      export TF_IGNORE_MAX_BAZEL_VERSION=1
-
-      # don't rebuild the world
-      export TF_SYSTEM_LIBS=${lib.concatStringsSep " " SYSLIBS}
-
-      export PYTHON_BIN_PATH="${python.interpreter}"
       export PYTHON_LIB_PATH="$NIX_BUILD_TOP/site-packages"
-      export TF_NEED_GCP=1
-      export TF_NEED_HDFS=1
-      export TF_ENABLE_XLA=${tfFeature xlaSupport}
-      export CC_OPT_FLAGS=" "
-      # https://github.com/tensorflow/tensorflow/issues/14454
-      export TF_NEED_MPI=${tfFeature cudaSupport}
-      export TF_NEED_CUDA=${tfFeature cudaSupport}
-      ${lib.optionalString cudaSupport ''
-        export TF_CUDA_PATHS="${cudatoolkit_joined},${cudnn},${nccl}"
-        export TF_CUDA_VERSION=${cudatoolkit.majorVersion}
-        export TF_CUDNN_VERSION=${cudnn.majorVersion}
-        export GCC_HOST_COMPILER_PATH=${bazel_cuda_toolkit}/bin/gcc
-        export GCC_HOST_COMPILER_PREFIX="${bazel_cuda_toolkit}/bin"
-        export TF_CUDA_COMPUTE_CAPABILITIES=${lib.concatStringsSep "," cudaCapabilities}
-      ''}
-
-      # https://github.com/tensorflow/tensorflow/issues/20919
-      sed -i '/androidndk/d' tensorflow/lite/kernels/internal/BUILD
-
       mkdir -p "$PYTHON_LIB_PATH"
     '';
 
-    # ar, cpp, gcov, strip, gcc, compat-ld, ld, objdump, dwp, nm, objcopy
     configurePhase = ''
       runHook preConfigure
-      # no flags (options provided by previously set environment variables)
-      export AR="${binutils.bintools}/bin/ar"
-      export BAZEL_TOOLS_OVERRIDE="ar=${binutils.bintools}/bin/ar"
-      # crosstool used with cuda
       ./configure
-
       runHook postConfigure
     '';
 
-    # FIXME
-    NIX_LDFLAGS = lib.optionals cudaSupport [ "-lcublas" "-lcudnn" "-lcuda" "-lcudart" ];
+    # FIXME: Tensorflow uses dlopen() for CUDA libraries.
+    # No idea why gpr isn't linked properly; perhaps Tensorflow expects a static library?
+    NIX_LDFLAGS = [ "-lgpr" ] ++ lib.optionals cudaSupport [ "-lcudart" "-lcublas" "-lcufft" "-lcurand" "-lcusolver" "-lcusparse" "-lcudnn" ];
 
-    hardeningDisable = [ "all" ];
+    hardeningDisable = [ "format" ];
 
     bazelFlags = [
       # temporary fixes to make the build work with bazel 0.27
@@ -250,7 +251,7 @@ let
       ++ lib.optional avx2Support "--copt=-mavx2"
       ++ lib.optional fmaSupport "--copt=-mfma";
 
-    bazelTarget = "//tensorflow/tools/pip_package:build_pip_package";
+    bazelTarget = "//tensorflow/tools/pip_package:build_pip_package //tensorflow/tools/lib_package:libtensorflow";
 
     fetchAttrs = {
       preInstall = ''
@@ -259,46 +260,36 @@ let
 
       # cudaSupport causes fetch of ncclArchive, resulting in different hashes
       sha256 = if cudaSupport then
-        "0q1pmw7fzn6l554ap576r48m0zgwb7n1ljhyy1p36708z94scdh4"
+        "0gmwwcld8hm1nfxmx2w2gqxjxq5b5ln7vbxn0x63x9yy7avs3dkp"
       else
-        "14n26h2r4w7wd5sddy4w0s51s2qcwf276n3hvv2505iysa8wqlc3";
+        "1d67h0n461cyfimzlsj6bv9p95iv9jhglr0izdc6y0n6w09z8pw0";
     };
 
     buildAttrs = {
+      outputs = [ "out" "python" ];
+
       preBuild = ''
         patchShebangs .
-
-        # beautiful bash to iterate over files containing a string
-        # https://github.com/bazelbuild/bazel/issues/5915#issuecomment-505100422
-        # .. to make sure the output directory is covered
-        grep -lrZ '/usr/bin/ar\b' .. | while IFS="" read -r -d "" file; do
-            # patch /usr/bin/ar to the proper location
-            echo "File is $file"
-            sed -i \
-              -e 's,/usr/bin/ar\b,${binutils.bintools}/bin/ar,g' \
-              "$file"
-        done
-
-
-        # Tensorboard pulls in a bunch of dependencies, some of which may
-        # include security vulnerabilities. So we make it optional.
-        # https://github.com/tensorflow/tensorflow/issues/20280#issuecomment-400230560
-        sed -i '/tensorboard >=/d' tensorflow/tools/pip_package/setup.py
       '';
 
-      # Could alternatively use --src instead of --dst to output the sources
-      # instead of a whee. Generating a wheel makes it easier to unify handling
-      # of source and binary build though.
       installPhase = ''
-        # work around timestamp issues
-        # https://github.com/NixOS/nixpkgs/issues/270#issuecomment-467583872
-        export SOURCE_DATE_EPOCH=315532800
-
-        # bulid the wheel, then move it to $out (building directly to $out
-        # would actually put it into a *directory* called $out, but we want the
-        # file itself in $out)
-        bazel-bin/tensorflow/tools/pip_package/build_pip_package --dst $PWD/dist
-        mv dist/*.whl "$out"
+        mkdir -p "$out"
+        tar -xf bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz -C "$out"
+        # Write pkgconfig file.
+        mkdir "$out/lib/pkgconfig"
+        cat > "$out/lib/pkgconfig/tensorflow.pc" << EOF
+        Name: TensorFlow
+        Version: ${version}
+        Description: Library for computation using data flow graphs for scalable machine learning
+        Requires:
+        Libs: -L$out/lib -ltensorflow
+        Cflags: -I$out/include/tensorflow
+        EOF
+
+        # build the source code, then copy it to $python (build_pip_package
+        # actually builds a symlink farm so we must dereference them).
+        bazel-bin/tensorflow/tools/pip_package/build_pip_package --src "$PWD/dist"
+        cp -Lr "$PWD/dist" "$python"
       '';
     };
   };
@@ -306,9 +297,7 @@ let
 in buildPythonPackage rec {
   inherit version pname;
 
-  src = bazel-wheel;
-
-  format = "wheel";
+  src = bazel-build.python;
 
   # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow
   # and the propagated input tensorflow-tensorboard, which causes environment collisions.
@@ -318,6 +307,14 @@ in buildPythonPackage rec {
     rm $out/bin/tensorboard
   '';
 
+  # TODO: remove after there's support for setupPyDistFlags
+  buildPhase = ''
+    runHook preBuild
+    cp ${setuppy} nix_run_setup
+    ${python.interpreter} nix_run_setup --project_name ${pname} bdist_wheel
+    runHook postBuild
+  '';
+
   # tensorflow/tools/pip_package/setup.py
   propagatedBuildInputs = [
     absl-py
@@ -349,6 +346,8 @@ in buildPythonPackage rec {
     ${python.interpreter} -c "import tensorflow"
   '';
 
+  passthru.libtensorflow = bazel-build.out;
+
   meta = with stdenv.lib; {
     description = "Computation using data flow graphs for scalable machine learning";
     homepage = http://tensorflow.org;
diff --git a/pkgs/development/python-modules/tensorflow/system-jsoncpp.patch b/pkgs/development/python-modules/tensorflow/system-jsoncpp.patch
new file mode 100644
index 000000000000..ecb2d04d7eef
--- /dev/null
+++ b/pkgs/development/python-modules/tensorflow/system-jsoncpp.patch
@@ -0,0 +1,21 @@
+diff --git a/third_party/systemlibs/jsoncpp.BUILD b/third_party/systemlibs/jsoncpp.BUILD
+index 526fd0c418..646f3fdcea 100644
+--- a/third_party/systemlibs/jsoncpp.BUILD
++++ b/third_party/systemlibs/jsoncpp.BUILD
+@@ -7,6 +7,7 @@ filegroup(
+ 
+ HEADERS = [
+     "include/json/autolink.h",
++    "include/json/allocator.h",
+     "include/json/config.h",
+     "include/json/features.h",
+     "include/json/forwards.h",
+@@ -23,7 +24,7 @@ genrule(
+     cmd = """
+       for i in $(OUTS); do
+         i=$${i##*/}
+-        ln -sf $(INCLUDEDIR)/jsoncpp/json/$$i $(@D)/include/json/$$i
++        ln -sf $(INCLUDEDIR)/json/$$i $(@D)/include/json/$$i
+       done
+     """,
+ )
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index a16369a1e072..55aa3e59f0a3 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -1093,12 +1093,15 @@ in
 
   tensor = libsForQt5.callPackage ../applications/networking/instant-messengers/tensor { };
 
-  libtensorflow = callPackage ../development/libraries/science/math/tensorflow/bin.nix {
-    cudaSupport = pkgs.config.cudaSupport or false;
-    inherit (linuxPackages) nvidia_x11;
-    cudatoolkit = cudatoolkit_10_0;
-    cudnn = cudnn_cudatoolkit_10_0;
-  };
+  libtensorflow =
+    if python.pkgs.tensorflow ? libtensorflow
+    then python.pkgs.tensorflow.libtensorflow
+    else callPackage ../development/libraries/science/math/tensorflow/bin.nix {
+      cudaSupport = pkgs.config.cudaSupport or false;
+      inherit (linuxPackages) nvidia_x11;
+      cudatoolkit = cudatoolkit_10_0;
+      cudnn = cudnn_cudatoolkit_10_0;
+    };
 
   behdad-fonts = callPackage ../data/fonts/behdad-fonts { };
 
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index c28779bf4433..429d26db0f16 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -5702,14 +5702,18 @@ in {
 
   tensorflow =
     if stdenv.isDarwin
-    then callPackage ../development/python-modules/tensorflow/bin.nix { }
+    then callPackage ../development/python-modules/tensorflow/bin.nix {
+      cudaSupport = pkgs.config.cudaSupport or false;
+      inherit (pkgs.linuxPackages) nvidia_x11;
+      cudatoolkit = pkgs.cudatoolkit_10;
+      cudnn = pkgs.cudnn_cudatoolkit_10;
+    }
     else callPackage ../development/python-modules/tensorflow rec {
       cudaSupport = pkgs.config.cudaSupport or false;
       inherit (pkgs.linuxPackages) nvidia_x11;
       cudatoolkit = pkgs.cudatoolkit_10;
       cudnn = pkgs.cudnn_cudatoolkit_10;
       nccl = pkgs.nccl_cudatoolkit_10;
-      protobuf_cc = pkgs.protobuf; # not the python version
     };
 
   tensorflowWithoutCuda = self.tensorflow.override {
-- 
cgit 1.4.1


From e458a340128834024d3445af5e40fecfbbfea086 Mon Sep 17 00:00:00 2001
From: Timo Kaufmann <timokau@zoho.com>
Date: Thu, 18 Jul 2019 13:49:00 +0200
Subject: buildBazelPackage: add flags for build and fetch

They sometimes take separate flags.
---
 pkgs/build-support/build-bazel-package/default.nix | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/pkgs/build-support/build-bazel-package/default.nix b/pkgs/build-support/build-bazel-package/default.nix
index a308eb8aaa4c..32cec42160f8 100644
--- a/pkgs/build-support/build-bazel-package/default.nix
+++ b/pkgs/build-support/build-bazel-package/default.nix
@@ -4,7 +4,7 @@
 , lib
 }:
 
-args@{ name, bazelFlags ? [], bazelTarget, buildAttrs, fetchAttrs, ... }:
+args@{ name, bazelFlags ? [], bazelBuildFlags ? [], bazelFetchFlags ? [], bazelTarget, buildAttrs, fetchAttrs, ... }:
 
 let
   fArgs = removeAttrs args [ "buildAttrs" "fetchAttrs" ];
@@ -12,11 +12,11 @@ let
   fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "sha256" ];
 
 in stdenv.mkDerivation (fBuildAttrs // {
-  inherit name bazelFlags bazelTarget;
+  inherit name bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget;
 
   deps = stdenv.mkDerivation (fFetchAttrs // {
     name = "${name}-deps";
-    inherit bazelFlags bazelTarget;
+    inherit bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget;
 
     nativeBuildInputs = fFetchAttrs.nativeBuildInputs or [] ++ [ bazel ];
 
@@ -49,6 +49,7 @@ in stdenv.mkDerivation (fBuildAttrs // {
         fetch \
         --loading_phase_threads=1 \
         $bazelFlags \
+        $bazelFetchFlags \
         $bazelTarget
 
       runHook postBuild
@@ -149,6 +150,7 @@ in stdenv.mkDerivation (fBuildAttrs // {
       "''${host_linkopts[@]}" \
       '' + ''
       $bazelFlags \
+      $bazelBuildFlags \
       $bazelTarget
 
     runHook postBuild
-- 
cgit 1.4.1


From d30ec1a0efce74768979ce253bfe3922d85e3503 Mon Sep 17 00:00:00 2001
From: Timo Kaufmann <timokau@zoho.com>
Date: Thu, 18 Jul 2019 13:49:30 +0200
Subject: python.pkgs.tensorflow: fix optimization flags

Now need to be passed in the configure phase.

abbradar: Don't change CUDA build hash.
---
 pkgs/development/python-modules/tensorflow/default.nix | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix
index d2d8d50fda46..a645e6d7909e 100644
--- a/pkgs/development/python-modules/tensorflow/default.nix
+++ b/pkgs/development/python-modules/tensorflow/default.nix
@@ -219,7 +219,12 @@ let
       sed -i '/tensorboard >=/d' tensorflow/tools/pip_package/setup.py
     '';
 
-    preConfigure = ''
+    preConfigure = let
+      opt_flags = []
+        ++ lib.optionals sse42Support ["-msse4.2"]
+        ++ lib.optionals avx2Support ["-mavx2"]
+        ++ lib.optionals fmaSupport ["-mfma"];
+    in ''
       patchShebangs configure
 
       # dummy ldconfig
@@ -229,6 +234,7 @@ let
       export PATH="$PWD/dummy-ldconfig:$PATH"
 
       export PYTHON_LIB_PATH="$NIX_BUILD_TOP/site-packages"
+      export CC_OPT_FLAGS="${lib.concatStringsSep " " opt_flags}"
       mkdir -p "$PYTHON_LIB_PATH"
     '';
 
@@ -247,9 +253,10 @@ let
     bazelFlags = [
       # temporary fixes to make the build work with bazel 0.27
       "--incompatible_no_support_tools_in_action_inputs=false"
-    ] ++ lib.optional sse42Support "--copt=-msse4.2"
-      ++ lib.optional avx2Support "--copt=-mavx2"
-      ++ lib.optional fmaSupport "--copt=-mfma";
+    ];
+    bazelBuildFlags = [
+      "--config=opt" # optimize using the flags set in the configure phase
+    ];
 
     bazelTarget = "//tensorflow/tools/pip_package:build_pip_package //tensorflow/tools/lib_package:libtensorflow";
 
@@ -260,7 +267,7 @@ let
 
       # cudaSupport causes fetch of ncclArchive, resulting in different hashes
       sha256 = if cudaSupport then
-        "0gmwwcld8hm1nfxmx2w2gqxjxq5b5ln7vbxn0x63x9yy7avs3dkp"
+        "07p6f3rbdw7y2ky5wrldjcvzdzhcynlmmydwb7cz12iphazhmipy"
       else
         "1d67h0n461cyfimzlsj6bv9p95iv9jhglr0izdc6y0n6w09z8pw0";
     };
-- 
cgit 1.4.1


From 19cdfe87f912a3491755dc65493542011325d311 Mon Sep 17 00:00:00 2001
From: Nikolay Amiantov <ab@fmap.me>
Date: Thu, 25 Jul 2019 16:00:22 +0300
Subject: python.pkgs.tensorflow: update dependencies hash

---
 pkgs/development/python-modules/tensorflow/default.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix
index a645e6d7909e..43134fb746d8 100644
--- a/pkgs/development/python-modules/tensorflow/default.nix
+++ b/pkgs/development/python-modules/tensorflow/default.nix
@@ -267,9 +267,9 @@ let
 
       # cudaSupport causes fetch of ncclArchive, resulting in different hashes
       sha256 = if cudaSupport then
-        "07p6f3rbdw7y2ky5wrldjcvzdzhcynlmmydwb7cz12iphazhmipy"
+        "1rbzvir569ch33bpvm47byx954vr80rjnzrbgs4rg3bqww73hr3k"
       else
-        "1d67h0n461cyfimzlsj6bv9p95iv9jhglr0izdc6y0n6w09z8pw0";
+        "1wnhma450zh72raiq7ddd9lyq3a056cjqiy6i7y4sidiy09ncvcg";
     };
 
     buildAttrs = {
-- 
cgit 1.4.1


From cd0e461f93972c754ed7cd0cd0129bc56a9fdbdd Mon Sep 17 00:00:00 2001
From: Nikolay Amiantov <ab@fmap.me>
Date: Wed, 31 Jul 2019 13:33:26 +0300
Subject: tensorflow: expose binary builds

---
 pkgs/top-level/all-packages.nix    | 14 ++++++++------
 pkgs/top-level/python-packages.nix | 31 ++++++++++++++++---------------
 2 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 55aa3e59f0a3..8d3f900b5a67 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -1093,15 +1093,17 @@ in
 
   tensor = libsForQt5.callPackage ../applications/networking/instant-messengers/tensor { };
 
+  libtensorflow-bin = callPackage ../development/libraries/science/math/tensorflow/bin.nix {
+    cudaSupport = pkgs.config.cudaSupport or false;
+    inherit (linuxPackages) nvidia_x11;
+    cudatoolkit = cudatoolkit_10_0;
+    cudnn = cudnn_cudatoolkit_10_0;
+  };
+
   libtensorflow =
     if python.pkgs.tensorflow ? libtensorflow
     then python.pkgs.tensorflow.libtensorflow
-    else callPackage ../development/libraries/science/math/tensorflow/bin.nix {
-      cudaSupport = pkgs.config.cudaSupport or false;
-      inherit (linuxPackages) nvidia_x11;
-      cudatoolkit = cudatoolkit_10_0;
-      cudnn = cudnn_cudatoolkit_10_0;
-    };
+    else libtensorflow-bin;
 
   behdad-fonts = callPackage ../data/fonts/behdad-fonts { };
 
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 429d26db0f16..68718fe9f6bc 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -5700,21 +5700,22 @@ in {
 
   tensorflow-tensorboard = callPackage ../development/python-modules/tensorflow-tensorboard { };
 
-  tensorflow =
-    if stdenv.isDarwin
-    then callPackage ../development/python-modules/tensorflow/bin.nix {
-      cudaSupport = pkgs.config.cudaSupport or false;
-      inherit (pkgs.linuxPackages) nvidia_x11;
-      cudatoolkit = pkgs.cudatoolkit_10;
-      cudnn = pkgs.cudnn_cudatoolkit_10;
-    }
-    else callPackage ../development/python-modules/tensorflow rec {
-      cudaSupport = pkgs.config.cudaSupport or false;
-      inherit (pkgs.linuxPackages) nvidia_x11;
-      cudatoolkit = pkgs.cudatoolkit_10;
-      cudnn = pkgs.cudnn_cudatoolkit_10;
-      nccl = pkgs.nccl_cudatoolkit_10;
-    };
+  tensorflow-bin = callPackage ../development/python-modules/tensorflow/bin.nix {
+    cudaSupport = pkgs.config.cudaSupport or false;
+    inherit (pkgs.linuxPackages) nvidia_x11;
+    cudatoolkit = pkgs.cudatoolkit_10;
+    cudnn = pkgs.cudnn_cudatoolkit_10;
+  };
+
+  tensorflow-build = callPackage ../development/python-modules/tensorflow {
+    cudaSupport = pkgs.config.cudaSupport or false;
+    inherit (pkgs.linuxPackages) nvidia_x11;
+    cudatoolkit = pkgs.cudatoolkit_10;
+    cudnn = pkgs.cudnn_cudatoolkit_10;
+    nccl = pkgs.nccl_cudatoolkit_10;
+  };
+
+  tensorflow = if stdenv.isDarwin then self.tensorflow-bin else self.tensorflow-build;
 
   tensorflowWithoutCuda = self.tensorflow.override {
     cudaSupport = false;
-- 
cgit 1.4.1