libtensorflow: add binary build and add automatic generation

4 files changed, 111 insertions, 88 deletions

diff --git a/pkgs/development/libraries/libtensorflow/default.nix b/pkgs/development/libraries/libtensorflow/default.nix
deleted file mode 100644
index e4f6a868e9fd..000000000000
--- a/pkgs/development/libraries/libtensorflow/default.nix
+++ /dev/null
@@ -1,88 +0,0 @@
-{ stdenv
-, fetchurl, python3, which
-, patchelf, fetchFromGitHub, buildBazelPackage
-, cudatoolkit, cudnn, nccl, nvidia_x11, symlinkJoin
-, binutils, gcc, binutils-unwrapped, gcc-unwrapped
-, glibcLocales
-, features ? ["sse4.2" "avx" "avx2" "fma"]
-, arch ? "x86-64"
-, cudaSupport ? false
-, computeCapabilities ? [] # example: [ "7.5" ], see https://developer.nvidia.com/cuda-gpus
-}:
-let
-  inherit (stdenv) lib;
-  cudatoolkit_joined = symlinkJoin {
-    name = "unsplit_cudatoolkit";
-    paths = [ cudatoolkit.out
-              cudatoolkit.lib ];};
-  gcc_joined = symlinkJoin {
-    name = "gcc-joined";
-    paths = [ binutils gcc binutils-unwrapped gcc-unwrapped ];
-  };
-in
-buildBazelPackage rec {
-  pname = "tensorflow";
-  version = "1.14.0";
-  name = "${pname}-${version}";
-  bazelFlags = [ "--incompatible_no_support_tools_in_action_inputs=false" ];
-  bazelTarget = "//tensorflow/tools/lib_package:libtensorflow";
-  fetchAttrs = {
-    sha256 = if cudaSupport then
-      "127xxwy3a2h1qsv2sqfhrh65g69hlb1q003vyyg7yjfqgfah9p2z"
-      else "1di1pnknr1hxdpn75lxf9c6dvb5kgllmgb9r9rgh5c2g9iil17zy";
-  };
-  CC_OPT_FLAGS = "-march=${arch} " + stdenv.lib.concatMapStringsSep " " (f: "-m"+f) features;
-  NIX_CFLAGS_COMPILE = CC_OPT_FLAGS;
-  TF_NEED_CUDA = if cudaSupport then "1" else "0";
-  TF_IGNORE_MAX_BAZEL_VERSION = "1";
-  TF_CUDA_COMPUTE_CAPABILITIES = stdenv.lib.concatStringsSep "," computeCapabilities;
-  nativeBuildInputs = [ glibcLocales python3 which ]
-    ++ (lib.optionals cudaSupport [
-    cudatoolkit_joined cudnn nvidia_x11
-  ]);
-  TF_CUDA_PATHS = lib.optionalString cudaSupport "${cudatoolkit_joined},${cudnn}";
-  src = fetchFromGitHub {
-    repo = "tensorflow";
-    owner = "tensorflow";
-    rev = "v${version}";
-    sha256 = "06jvwlsm14b8rqwd8q8796r0vmn0wk64s4ps2zg0sapkmp9vvcmi";
-  };
-  prePatch = ''
-    # doesn't work:
-    sed -i '/saved_model_portable_proto/d' tensorflow/cc/saved_model/BUILD
-    # calls ldconfig -p:
-    sed -i 's/+ _get_ld_config_paths()//' third_party/gpus/find_cuda_config.py
-    patchShebangs ./configure
-    patchShebangs tensorflow/tools/lib_package/concat_licenses.sh
-    patchShebangs third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl
-  '';
-  GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${gcc_joined}/bin";
-  configurePhase = ''
-    runHook preConfigure
-    ./configure
-    runHook postConfigure
-  '';
-  buildAttrs = {
-    installPhase = ''
-      mkdir -p $out
-      tar -xf bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz -C $out
-      # Write pkgconfig file.
-      mkdir $out/lib/pkgconfig
-      cat > $out/lib/pkgconfig/tensorflow.pc << EOF
-      Name: TensorFlow
-      Version: ${version}
-      Description: Library for computation using data flow graphs for scalable machine learning
-      Requires:
-      Libs: -L$out/lib -ltensorflow
-      Cflags: -I$out/include/tensorflow
-      EOF
-    '';
-  };
-  meta = with lib; {
-    description = "C API for TensorFlow";
-    homepage = https://www.tensorflow.org/versions/master/install/install_c;
-    license = licenses.asl20;
-    platforms = with platforms; linux ++ darwin;
-    maintainers = with maintainers; [ basvandijk yorickvp ];
-  };
-}
diff --git a/pkgs/development/libraries/science/math/tensorflow/bin.nix b/pkgs/development/libraries/science/math/tensorflow/bin.nix
new file mode 100644
index 000000000000..3db7a6f1d1e2
--- /dev/null
+++ b/pkgs/development/libraries/science/math/tensorflow/bin.nix
@@ -0,0 +1,72 @@
+{ stdenv
+, fetchurl
+, patchelf
+, cudaSupport ? false, symlinkJoin, cudatoolkit, cudnn, nvidia_x11
+}:
+
+with stdenv.lib;
+let
+  unavailable = throw "libtensorflow is not available for this platform!";
+
+  tfType = if cudaSupport then "gpu" else "cpu";
+
+  system = 
+    if      stdenv.isLinux  then "linux"
+    else if stdenv.isDarwin then "darwin"
+    else unavailable;
+
+  platform =
+    if stdenv.isx86_64 then "x86_64"
+    else unavailable;
+
+  rpath = makeLibraryPath ([stdenv.cc.libc stdenv.cc.cc.lib] ++
+            optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]);
+
+  packages = import ./binary-hashes.nix;
+  packageName = "${tfType}-${system}-${platform}";
+  url = packages.${packageName} or unavailable;
+
+  patchLibs =
+    if stdenv.isDarwin
+    then ''
+      install_name_tool -id $out/lib/libtensorflow.dylib $out/lib/libtensorflow.dylib
+      install_name_tool -id $out/lib/libtensorflow_framework.dylib $out/lib/libtensorflow_framework.dylib
+    ''
+    else ''
+      patchelf --set-rpath "${rpath}:$out/lib" $out/lib/libtensorflow.so
+      patchelf --set-rpath "${rpath}" $out/lib/libtensorflow_framework.so
+    '';
+
+in stdenv.mkDerivation rec {
+  pname = "libtensorflow";
+  inherit (packages) version;
+
+  src = fetchurl url;
+
+  # Patch library to use our libc, libstdc++ and others
+  buildCommand = ''
+    mkdir -pv $out
+    tar -C $out -xzf $src
+    chmod -R +w $out
+    ${patchLibs}
+
+    # Write pkgconfig file.
+    mkdir $out/lib/pkgconfig
+    cat > $out/lib/pkgconfig/tensorflow.pc << EOF
+    Name: TensorFlow
+    Version: ${version}
+    Description: Library for computation using data flow graphs for scalable machine learning
+    Requires:
+    Libs: -L$out/lib -ltensorflow
+    Cflags: -I$out/include/tensorflow
+    EOF
+  '';
+
+  meta = {
+    description = "C API for TensorFlow";
+    homepage = https://www.tensorflow.org/install/lang_c;
+    license = licenses.asl20;
+    platforms = [ "x86_64-linux" "x86_64-darwin" ];
+    maintainers = with maintainers; [ basvandijk ];
+  };
+}
diff --git a/pkgs/development/libraries/science/math/tensorflow/binary-hashes.nix b/pkgs/development/libraries/science/math/tensorflow/binary-hashes.nix
new file mode 100644
index 000000000000..892dfa2a6097
--- /dev/null
+++ b/pkgs/development/libraries/science/math/tensorflow/binary-hashes.nix
@@ -0,0 +1,15 @@
+{
+version = "1.14.0";
+"cpu-linux-x86_64" = {
+  url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz";
+  sha256 = "04bi3ijq4sbb8c5vk964zlv0j9mrjnzzxd9q9knq3h273nc1a36k";
+};
+"gpu-linux-x86_64" = {
+  url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-1.14.0.tar.gz";
+  sha256 = "1ffnpyj9jjgwxpjfiyjvq4dm3n6nwiksim5jld9zw7fdswh215x6";
+};
+"cpu-darwin-x86_64" = {
+  url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-darwin-x86_64-1.14.0.tar.gz";
+  sha256 = "0zsd5ils1a17j6jzh0c7q1z56fw46gkzybbnms7h2rgg8al0rh92";
+};
+}
diff --git a/pkgs/development/libraries/science/math/tensorflow/prefetcher.sh b/pkgs/development/libraries/science/math/tensorflow/prefetcher.sh
new file mode 100755
index 000000000000..515f25df4c1f
--- /dev/null
+++ b/pkgs/development/libraries/science/math/tensorflow/prefetcher.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+version=1.14.0
+hashfile=binary-hashes.nix
+rm -f $hashfile
+echo "{" >> $hashfile
+echo "version = \"$version\";" >> $hashfile
+for sys in "linux" "darwin"; do
+    for tfpref in "cpu" "gpu"; do
+        for platform in "x86_64"; do
+            if [ $sys = "darwin" ] && [ $tfpref = "gpu" ]; then
+               continue
+            fi
+            url=https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-$tfpref-$sys-$platform-$version.tar.gz
+            hash=$(nix-prefetch-url $url)
+            echo "\"${tfpref}-${sys}-${platform}\" = {" >> $hashfile
+            echo "  url = \"$url\";" >> $hashfile
+            echo "  sha256 = \"$hash\";" >> $hashfile
+            echo "};" >> $hashfile
+        done
+    done
+done
+echo "}" >> $hashfile
+