From 9939bd67f02f08383d683f1bc9423040a730ab84 Mon Sep 17 00:00:00 2001 From: "(cdep)illabout" Date: Fri, 28 Feb 2020 18:15:28 +0900 Subject: python3Packages.tensorflow-estimator: 1.15.1 -> 2.1.0 (cherry picked from commit c186f5ec4d5148b95ac535ee6affb43bd1de9bc2) --- pkgs/development/python-modules/tensorflow-estimator/default.nix | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkgs/development/python-modules/tensorflow-estimator/default.nix b/pkgs/development/python-modules/tensorflow-estimator/default.nix index 9f6be4d1ddac..47ce0f83b02d 100644 --- a/pkgs/development/python-modules/tensorflow-estimator/default.nix +++ b/pkgs/development/python-modules/tensorflow-estimator/default.nix @@ -1,19 +1,18 @@ { stdenv, fetchPypi, buildPythonPackage , numpy -, absl-py +, absl-py , mock }: buildPythonPackage rec { pname = "tensorflow-estimator"; - # This is effectively 1.15.0. Upstream tagged 1.15.0 by mistake before actually updating the version in setup.py, which is why this tag is called 1.15.1. - version = "1.15.1"; + version = "2.1.0"; format = "wheel"; src = fetchPypi { pname = "tensorflow_estimator"; inherit version format; - sha256 = "1fc61wmc0w22frs79j2x4g6wnv5g21xc6rix1g4bsvy9qfvvylw8"; + sha256 = "0wk9viil54ms1s2ir7zxygqa425i69hx8zngwhdqvw9nlr4gdig5"; }; propagatedBuildInputs = [ mock numpy absl-py ]; -- cgit 1.4.1 From 6c7b2ac58e343601d999f82c17ab77dba876fdd7 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Mon, 2 Mar 2020 17:22:35 +0100 Subject: pythonPackages.tensorflow-estimator_1_15_1: init at 1.15.1 TF 1.15 still needs an older version of the tensorflow-estimator package. (cherry picked from commit c539f937c539b99e8c8b763b2568b52e39d3e9c4) --- .../python-modules/tensorflow-estimator/1_15_1.nix | 27 ++++++++++++++++++++++ .../python-modules/tensorflow/default.nix | 4 ++-- pkgs/top-level/python-packages.nix | 1 + 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 pkgs/development/python-modules/tensorflow-estimator/1_15_1.nix diff --git a/pkgs/development/python-modules/tensorflow-estimator/1_15_1.nix b/pkgs/development/python-modules/tensorflow-estimator/1_15_1.nix new file mode 100644 index 000000000000..39c667357c8f --- /dev/null +++ b/pkgs/development/python-modules/tensorflow-estimator/1_15_1.nix @@ -0,0 +1,27 @@ +{ stdenv, fetchPypi, buildPythonPackage +, numpy +, absl-py +, mock +}: + +buildPythonPackage rec { + pname = "tensorflow-estimator"; + version = "1.15.1"; + format = "wheel"; + + src = fetchPypi { + pname = "tensorflow_estimator"; + inherit version format; + sha256 = "1fc61wmc0w22frs79j2x4g6wnv5g21xc6rix1g4bsvy9qfvvylw8"; + }; + + propagatedBuildInputs = [ mock numpy absl-py ]; + + meta = with stdenv.lib; { + description = "TensorFlow Estimator is a high-level API that encapsulates model training, evaluation, prediction, and exporting."; + homepage = http://tensorflow.org; + license = licenses.asl20; + maintainers = with maintainers; [ jyp ]; + }; +} + diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 54629bebc116..7c80452b20fd 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -7,7 +7,7 @@ , future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta , functools32 , opt-einsum -, termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator +, termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator_1_15_1 # Common deps , git, swig, which, binutils, glibcLocales, cython # Common libraries @@ -373,7 +373,7 @@ in buildPythonPackage { numpy six protobuf - tensorflow-estimator + tensorflow-estimator_1_15_1 termcolor wrapt grpcio diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 59ac9274b3ec..648b815e9ea2 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -6549,6 +6549,7 @@ in { zerobin = callPackage ../development/python-modules/zerobin { }; tensorflow-estimator = callPackage ../development/python-modules/tensorflow-estimator { }; + tensorflow-estimator_1_15_1 = callPackage ../development/python-modules/tensorflow-estimator/1_15_1.nix { }; tensorflow-probability = callPackage ../development/python-modules/tensorflow-probability { }; -- cgit 1.4.1 From d4de62dedd118b1e482307a1c8561985fee0fe01 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Mar 2020 04:28:34 +0000 Subject: bazel_0_29: init at 0.29.1 Needed for Tensorflow 2.1 --- .../python-modules/tensorflow/default.nix | 4 +- .../tools/build-managers/bazel/bazel_0/default.nix | 375 -------------- .../tools/build-managers/bazel/bazel_0/glibc.patch | 78 --- .../bazel/bazel_0/python-stub-path-fix.patch | 13 - .../build-managers/bazel/bazel_0_26/default.nix | 375 ++++++++++++++ .../build-managers/bazel/bazel_0_26/glibc.patch | 78 +++ .../bazel/bazel_0_26/python-stub-path-fix.patch | 13 + .../build-managers/bazel/bazel_0_29/default.nix | 559 +++++++++++++++++++++ .../build-managers/bazel/bazel_0_29/glibc.patch | 78 +++ .../build-managers/bazel/bazel_0_29/src-deps.json | 506 +++++++++++++++++++ pkgs/top-level/all-packages.nix | 13 +- 11 files changed, 1623 insertions(+), 469 deletions(-) delete mode 100644 pkgs/development/tools/build-managers/bazel/bazel_0/default.nix delete mode 100644 pkgs/development/tools/build-managers/bazel/bazel_0/glibc.patch delete mode 100644 pkgs/development/tools/build-managers/bazel/bazel_0/python-stub-path-fix.patch create mode 100644 pkgs/development/tools/build-managers/bazel/bazel_0_26/default.nix create mode 100644 pkgs/development/tools/build-managers/bazel/bazel_0_26/glibc.patch create mode 100644 pkgs/development/tools/build-managers/bazel/bazel_0_26/python-stub-path-fix.patch create mode 100644 pkgs/development/tools/build-managers/bazel/bazel_0_29/default.nix create mode 100644 pkgs/development/tools/build-managers/bazel/bazel_0_29/glibc.patch create mode 100644 pkgs/development/tools/build-managers/bazel/bazel_0_29/src-deps.json diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 7c80452b20fd..eafc5ba7447a 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -1,4 +1,4 @@ -{ stdenv, pkgs, bazel_0, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin +{ stdenv, pkgs, bazel_0_26, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin , addOpenGLRunpath # Python deps , buildPythonPackage, isPy3k, isPy27, pythonOlder, pythonAtLeast, python @@ -94,7 +94,7 @@ let bazel-build = buildBazelPackage { name = "${pname}-${version}"; - bazel = bazel_0; + bazel = bazel_0_26; src = fetchFromGitHub { owner = "tensorflow"; diff --git a/pkgs/development/tools/build-managers/bazel/bazel_0/default.nix b/pkgs/development/tools/build-managers/bazel/bazel_0/default.nix deleted file mode 100644 index f54868de533c..000000000000 --- a/pkgs/development/tools/build-managers/bazel/bazel_0/default.nix +++ /dev/null @@ -1,375 +0,0 @@ -{ stdenv, callPackage, lib, fetchurl, fetchpatch, runCommand, makeWrapper -, zip, unzip, bash, writeCBin, coreutils -, which, python, perl, gawk, gnused, gnutar, gnugrep, gzip, findutils -# Apple dependencies -, cctools, clang, libcxx, CoreFoundation, CoreServices, Foundation -# Allow to independently override the jdks used to build and run respectively -, buildJdk, runJdk -, buildJdkName -, runtimeShell -# Always assume all markers valid (don't redownload dependencies). -# Also, don't clean up environment variables. -, enableNixHacks ? false -}: - -let - srcDeps = [ - # From: $REPO_ROOT/WORKSPACE - (fetchurl { - url = "https://github.com/google/desugar_jdk_libs/archive/915f566d1dc23bc5a8975320cd2ff71be108eb9c.zip"; - sha256 = "0b926df7yxyyyiwm9cmdijy6kplf0sghm23sf163zh8wrk87wfi7"; - }) - (fetchurl { - url = "https://mirror.bazel.build/github.com/bazelbuild/skydoc/archive/2d9566b21fbe405acf5f7bf77eda30df72a4744c.tar.gz"; - sha256 = "4a1318fed4831697b83ce879b3ab70ae09592b167e5bda8edaff45132d1c3b3f"; - }) - (fetchurl { - url = "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/archive/f83cb8dd6f5658bc574ccd873e25197055265d1c.tar.gz"; - sha256 = "ba5d15ca230efca96320085d8e4d58da826d1f81b444ef8afccd8b23e0799b52"; - }) - (fetchurl { - url = "https://mirror.bazel.build/github.com/bazelbuild/rules_sass/archive/8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz"; - sha256 = "d868ce50d592ef4aad7dec4dd32ae68d2151261913450fac8390b3fd474bb898"; - }) - (fetchurl { - url = "https://mirror.bazel.build/bazel_java_tools/releases/javac10/v3.1/java_tools_javac10_linux-v3.1.zip"; - sha256 = "a0cd51f9db1bf05a722ff7f5c60a07fa1c7d27428fff0815c342d32aa6c53576"; - }) - (fetchurl { - url = "https://mirror.bazel.build/bazel_java_tools/releases/javac10/v3.1/java_tools_javac10_darwin-v3.1.zip"; - sha256 = "c646aad8808b8ec5844d6a80a1287fc8e13203375fe40d6af4819eff48b9bbaf"; - }) - (fetchurl { - url = "https://mirror.bazel.build/bazel_coverage_output_generator/releases/coverage_output_generator-v1.0.zip"; - sha256 = "cc470e529fafb6165b5be3929ff2d99b38429b386ac100878687416603a67889"; - }) - (fetchurl { - url = "https://github.com/bazelbuild/rules_nodejs/archive/0.16.2.zip"; - sha256 = "9b72bb0aea72d7cbcfc82a01b1e25bf3d85f791e790ddec16c65e2d906382ee0"; - }) - (fetchurl { - url = "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.2.tar.gz"; - sha256 = "04f85f2dd049e87805511e3babc5cea3f5e72332b1627e34f3a5461cc38e815f"; - }) - ]; - - distDir = runCommand "bazel-deps" {} '' - mkdir -p $out - for i in ${builtins.toString srcDeps}; do cp $i $out/$(stripHash $i); done - ''; - - defaultShellPath = lib.makeBinPath - # Keep this list conservative. For more exotic tools, prefer to use - # @rules_nixpkgs to pull in tools from the nix repository. Example: - # - # WORKSPACE: - # - # nixpkgs_git_repository( - # name = "nixpkgs", - # revision = "def5124ec8367efdba95a99523dd06d918cb0ae8", - # ) - # - # # This defines an external Bazel workspace. - # nixpkgs_package( - # name = "bison", - # repositories = { "nixpkgs": "@nixpkgs//:default.nix" }, - # ) - # - # some/BUILD.bazel: - # - # genrule( - # ... - # cmd = "$(location @bison//:bin/bison) -other -args", - # tools = [ - # ... - # "@bison//:bin/bison", - # ], - # ) - # - [ bash coreutils findutils gawk gnugrep gnutar gnused gzip which unzip ]; - - # Java toolchain used for the build and tests - javaToolchain = "@bazel_tools//tools/jdk:toolchain_host${buildJdkName}"; - -in -stdenv.mkDerivation rec { - - version = "0.26.0"; - - meta = with lib; { - homepage = "https://github.com/bazelbuild/bazel/"; - description = "Build tool that builds code quickly and reliably"; - license = licenses.asl20; - platforms = platforms.linux ++ platforms.darwin; - }; - - # Additional tests that check bazel’s functionality. Execute - # - # nix-build . -A bazel.tests - # - # in the nixpkgs checkout root to exercise them locally. - passthru.tests = { - pythonBinPath = callPackage ./python-bin-path-test.nix {}; - bashTools = callPackage ./bash-tools-test.nix {}; - }; - - name = "bazel-${version}"; - - src = fetchurl { - url = "https://github.com/bazelbuild/bazel/releases/download/${version}/${name}-dist.zip"; - sha256 = "d26dadf62959255d58e523da3448a6222af768fe1224e321b120c1d5bbe4b4f2"; - }; - - # Necessary for the tests to pass on Darwin with sandbox enabled. - # Bazel starts a local server and needs to bind a local address. - __darwinAllowLocalNetworking = true; - - sourceRoot = "."; - - patches = [ - ./glibc.patch - ./python-stub-path-fix.patch - ] ++ lib.optional enableNixHacks ../nix-hacks.patch; - - # Bazel expects several utils to be available in Bash even without PATH. Hence this hack. - - customBash = writeCBin "bash" '' - #include - #include - #include - #include - - extern char **environ; - - int main(int argc, char *argv[]) { - char *path = getenv("PATH"); - char *pathToAppend = "${defaultShellPath}"; - char *newPath; - if (path != NULL) { - int length = strlen(path) + 1 + strlen(pathToAppend) + 1; - newPath = malloc(length * sizeof(char)); - snprintf(newPath, length, "%s:%s", path, pathToAppend); - } else { - newPath = pathToAppend; - } - setenv("PATH", newPath, 1); - execve("${bash}/bin/bash", argv, environ); - return 0; - } - ''; - - postPatch = let - - darwinPatches = '' - # Disable Bazel's Xcode toolchain detection which would configure compilers - # and linkers from Xcode instead of from PATH - export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 - - # Explicitly configure gcov since we don't have it on Darwin, so autodetection fails - export GCOV=${coreutils}/bin/false - - # Framework search paths aren't added by bintools hook - # https://github.com/NixOS/nixpkgs/pull/41914 - export NIX_LDFLAGS="$NIX_LDFLAGS -F${CoreFoundation}/Library/Frameworks -F${CoreServices}/Library/Frameworks -F${Foundation}/Library/Frameworks" - - # libcxx includes aren't added by libcxx hook - # https://github.com/NixOS/nixpkgs/pull/41589 - export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -isystem ${libcxx}/include/c++/v1" - - # don't use system installed Xcode to run clang, use Nix clang instead - sed -i -e "s;/usr/bin/xcrun clang;${stdenv.cc}/bin/clang $NIX_CFLAGS_COMPILE $NIX_LDFLAGS -framework CoreFoundation;g" \ - scripts/bootstrap/compile.sh \ - src/tools/xcode/realpath/BUILD \ - src/tools/xcode/stdredirect/BUILD \ - tools/osx/BUILD - - # nixpkgs's libSystem cannot use pthread headers directly, must import GCD headers instead - sed -i -e "/#include /i #include " src/main/cpp/blaze_util_darwin.cc - - # clang installed from Xcode has a compatibility wrapper that forwards - # invocations of gcc to clang, but vanilla clang doesn't - sed -i -e 's;_find_generic(repository_ctx, "gcc", "CC", overriden_tools);_find_generic(repository_ctx, "clang", "CC", overriden_tools);g' tools/cpp/unix_cc_configure.bzl - - sed -i -e 's;/usr/bin/libtool;${cctools}/bin/libtool;g' tools/cpp/unix_cc_configure.bzl - wrappers=( tools/cpp/osx_cc_wrapper.sh tools/cpp/osx_cc_wrapper.sh.tpl ) - for wrapper in "''${wrappers[@]}"; do - sed -i -e "s,/usr/bin/install_name_tool,${cctools}/bin/install_name_tool,g" $wrapper - done - ''; - - genericPatches = '' - # Substitute python's stub shebang to plain python path. (see TODO add pr URL) - # See also `postFixup` where python is added to $out/nix-support - substituteInPlace src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt\ - --replace "/usr/bin/env python" "${python}/bin/python" \ - --replace "NIX_STORE_PYTHON_PATH" "${python}/bin/python" \ - - # md5sum is part of coreutils - sed -i 's|/sbin/md5|md5sum|' \ - src/BUILD - - # substituteInPlace is rather slow, so prefilter the files with grep - grep -rlZ /bin src/main/java/com/google/devtools | while IFS="" read -r -d "" path; do - # If you add more replacements here, you must change the grep above! - # Only files containing /bin are taken into account. - substituteInPlace "$path" \ - --replace /bin/bash ${customBash}/bin/bash \ - --replace /usr/bin/env ${coreutils}/bin/env \ - --replace /bin/true ${coreutils}/bin/true - done - - # Fixup scripts that generate scripts. Not fixed up by patchShebangs below. - substituteInPlace scripts/bootstrap/compile.sh \ - --replace /bin/bash ${customBash}/bin/bash - - # add nix environment vars to .bazelrc - cat >> .bazelrc <> runfiles.bash.tmp - cat tools/bash/runfiles/runfiles.bash >> runfiles.bash.tmp - mv runfiles.bash.tmp tools/bash/runfiles/runfiles.bash - - patchShebangs . - ''; - in lib.optionalString stdenv.hostPlatform.isDarwin darwinPatches - + genericPatches; - - buildInputs = [ - buildJdk - ]; - - # when a command can’t be found in a bazel build, you might also - # need to add it to `defaultShellPath`. - nativeBuildInputs = [ - zip - python - unzip - makeWrapper - which - customBash - ] ++ lib.optionals (stdenv.isDarwin) [ cctools libcxx CoreFoundation CoreServices Foundation ]; - - # Bazel makes extensive use of symlinks in the WORKSPACE. - # This causes problems with infinite symlinks if the build output is in the same location as the - # Bazel WORKSPACE. This is why before executing the build, the source code is moved into a - # subdirectory. - # Failing to do this causes "infinite symlink expansion detected" - preBuildPhases = ["preBuildPhase"]; - preBuildPhase = '' - mkdir bazel_src - shopt -s dotglob extglob - mv !(bazel_src) bazel_src - ''; - - buildPhase = '' - # Increasing memory during compilation might be necessary. - # export BAZEL_JAVAC_OPTS="-J-Xmx2g -J-Xms200m" - ./bazel_src/compile.sh - ./bazel_src/scripts/generate_bash_completion.sh \ - --bazel=./bazel_src/output/bazel \ - --output=./bazel_src/output/bazel-complete.bash \ - --prepend=./bazel_src/scripts/bazel-complete-header.bash \ - --prepend=./bazel_src/scripts/bazel-complete-template.bash - ''; - - installPhase = '' - mkdir -p $out/bin - - # official wrapper scripts that searches for $WORKSPACE_ROOT/tools/bazel - # if it can’t find something in tools, it calls $out/bin/bazel-real - cp ./bazel_src/scripts/packages/bazel.sh $out/bin/bazel - mv ./bazel_src/output/bazel $out/bin/bazel-real - - wrapProgram "$out/bin/bazel" --add-flags --server_javabase="${runJdk}" - - # shell completion files - mkdir -p $out/share/bash-completion/completions $out/share/zsh/site-functions - mv ./bazel_src/output/bazel-complete.bash $out/share/bash-completion/completions/bazel - cp ./bazel_src/scripts/zsh_completion/_bazel $out/share/zsh/site-functions/ - ''; - - # Temporarily disabling for now. A new approach is needed for this derivation as Bazel - # accesses the internet during the tests which fails in a sandbox. - doInstallCheck = false; - installCheckPhase = '' - export TEST_TMPDIR=$(pwd) - - hello_test () { - $out/bin/bazel test \ - --test_output=errors \ - --java_toolchain='${javaToolchain}' \ - examples/cpp:hello-success_test \ - examples/java-native/src/test/java/com/example/myproject:hello - } - - cd ./bazel_src - - # test whether $WORKSPACE_ROOT/tools/bazel works - - mkdir -p tools - cat > tools/bazel <<"EOF" - #!${runtimeShell} -e - exit 1 - EOF - chmod +x tools/bazel - - # first call should fail if tools/bazel is used - ! hello_test - - cat > tools/bazel <<"EOF" - #!${runtimeShell} -e - exec "$BAZEL_REAL" "$@" - EOF - - # second call succeeds because it defers to $out/bin/bazel-real - hello_test - ''; - - # Save paths to hardcoded dependencies so Nix can detect them. - postFixup = '' - mkdir -p $out/nix-support - echo "${customBash} ${defaultShellPath}" >> $out/nix-support/depends - # The templates get tar’d up into a .jar, - # so nix can’t detect python is needed in the runtime closure - echo "${python}" >> $out/nix-support/depends - ''; - - dontStrip = true; - dontPatchELF = true; -} diff --git a/pkgs/development/tools/build-managers/bazel/bazel_0/glibc.patch b/pkgs/development/tools/build-managers/bazel/bazel_0/glibc.patch deleted file mode 100644 index c4de48068f11..000000000000 --- a/pkgs/development/tools/build-managers/bazel/bazel_0/glibc.patch +++ /dev/null @@ -1,78 +0,0 @@ -From https://github.com/grpc/grpc/commit/57586a1ca7f17b1916aed3dea4ff8de872dbf853 -From: Benjamin Peterson -Date: Fri, 3 May 2019 08:11:00 -0700 -Subject: [PATCH] Rename gettid() functions. - -glibc 2.30 will declare its own gettid; see https://sourceware.org/git/?p=glibc.git;a=commit;h=1d0fc213824eaa2a8f8c4385daaa698ee8fb7c92. Rename the grpc versions to avoid naming conflicts. ---- - src/core/lib/gpr/log_linux.cc | 6 ++---- - src/core/lib/gpr/log_posix.cc | 4 ++-- - src/core/lib/iomgr/ev_epollex_linux.cc | 4 ++-- - 3 files changed, 6 insertions(+), 8 deletions(-) - -diff --git a/third_party/grpc/src/core/lib/gpr/log_linux.cc b/src/core/lib/gpr/log_linux.cc -index 81026e5689b..8b597b4cf2f 100644 ---- a/third_party/grpc/src/core/lib/gpr/log_linux.cc -+++ b/third_party/grpc/src/core/lib/gpr/log_linux.cc -@@ -40,7 +40,7 @@ - #include - #include - --static long gettid(void) { return syscall(__NR_gettid); } -+static long sys_gettid(void) { return syscall(__NR_gettid); } - - void gpr_log(const char* file, int line, gpr_log_severity severity, - const char* format, ...) { -@@ -70,7 +70,7 @@ void gpr_default_log(gpr_log_func_args* args) { - gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME); - struct tm tm; - static __thread long tid = 0; -- if (tid == 0) tid = gettid(); -+ if (tid == 0) tid = sys_gettid(); - - timer = static_cast(now.tv_sec); - final_slash = strrchr(args->file, '/'); -diff --git a/third_party/grpc/src/core/lib/gpr/log_posix.cc b/src/core/lib/gpr/log_posix.cc -index b6edc14ab6b..2f7c6ce3760 100644 ---- a/third_party/grpc/src/core/lib/gpr/log_posix.cc -+++ b/third_party/grpc/src/core/lib/gpr/log_posix.cc -@@ -31,7 +31,7 @@ - #include - #include - --static intptr_t gettid(void) { return (intptr_t)pthread_self(); } -+static intptr_t sys_gettid(void) { return (intptr_t)pthread_self(); } - - void gpr_log(const char* file, int line, gpr_log_severity severity, - const char* format, ...) { -@@ -86,7 +86,7 @@ void gpr_default_log(gpr_log_func_args* args) { - char* prefix; - gpr_asprintf(&prefix, "%s%s.%09d %7" PRIdPTR " %s:%d]", - gpr_log_severity_string(args->severity), time_buffer, -- (int)(now.tv_nsec), gettid(), display_file, args->line); -+ (int)(now.tv_nsec), sys_gettid(), display_file, args->line); - - fprintf(stderr, "%-70s %s\n", prefix, args->message); - gpr_free(prefix); -diff --git a/third_party/grpc/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc -index c2d80c08ddb..4a83cb6c215 100644 ---- a/third_party/grpc/src/core/lib/iomgr/ev_epollex_linux.cc -+++ b/third_party/grpc/src/core/lib/iomgr/ev_epollex_linux.cc -@@ -1077,7 +1077,7 @@ static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, - } - - #ifndef NDEBUG --static long gettid(void) { return syscall(__NR_gettid); } -+static long sys_gettid(void) { return syscall(__NR_gettid); } - #endif - - /* pollset->mu lock must be held by the caller before calling this. -@@ -1097,7 +1097,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset, - #define WORKER_PTR (&worker) - #endif - #ifndef NDEBUG -- WORKER_PTR->originator = gettid(); -+ WORKER_PTR->originator = sys_gettid(); - #endif - if (GRPC_TRACE_FLAG_ENABLED(grpc_polling_trace)) { - gpr_log(GPR_INFO, diff --git a/pkgs/development/tools/build-managers/bazel/bazel_0/python-stub-path-fix.patch b/pkgs/development/tools/build-managers/bazel/bazel_0/python-stub-path-fix.patch deleted file mode 100644 index cbc4192d2d9b..000000000000 --- a/pkgs/development/tools/build-managers/bazel/bazel_0/python-stub-path-fix.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt -index dac21c9a83..69b11c283f 100644 ---- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt -+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt -@@ -67,7 +67,7 @@ def FindPythonBinary(module_space): - return os.path.join(module_space, PYTHON_BINARY) - else: - # Case 4: Path has to be looked up in the search path. -- return SearchPath(PYTHON_BINARY) -+ return "NIX_STORE_PYTHON_PATH" - - def CreatePythonPathEntries(python_imports, module_space): - parts = python_imports.split(':'); diff --git a/pkgs/development/tools/build-managers/bazel/bazel_0_26/default.nix b/pkgs/development/tools/build-managers/bazel/bazel_0_26/default.nix new file mode 100644 index 000000000000..f54868de533c --- /dev/null +++ b/pkgs/development/tools/build-managers/bazel/bazel_0_26/default.nix @@ -0,0 +1,375 @@ +{ stdenv, callPackage, lib, fetchurl, fetchpatch, runCommand, makeWrapper +, zip, unzip, bash, writeCBin, coreutils +, which, python, perl, gawk, gnused, gnutar, gnugrep, gzip, findutils +# Apple dependencies +, cctools, clang, libcxx, CoreFoundation, CoreServices, Foundation +# Allow to independently override the jdks used to build and run respectively +, buildJdk, runJdk +, buildJdkName +, runtimeShell +# Always assume all markers valid (don't redownload dependencies). +# Also, don't clean up environment variables. +, enableNixHacks ? false +}: + +let + srcDeps = [ + # From: $REPO_ROOT/WORKSPACE + (fetchurl { + url = "https://github.com/google/desugar_jdk_libs/archive/915f566d1dc23bc5a8975320cd2ff71be108eb9c.zip"; + sha256 = "0b926df7yxyyyiwm9cmdijy6kplf0sghm23sf163zh8wrk87wfi7"; + }) + (fetchurl { + url = "https://mirror.bazel.build/github.com/bazelbuild/skydoc/archive/2d9566b21fbe405acf5f7bf77eda30df72a4744c.tar.gz"; + sha256 = "4a1318fed4831697b83ce879b3ab70ae09592b167e5bda8edaff45132d1c3b3f"; + }) + (fetchurl { + url = "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/archive/f83cb8dd6f5658bc574ccd873e25197055265d1c.tar.gz"; + sha256 = "ba5d15ca230efca96320085d8e4d58da826d1f81b444ef8afccd8b23e0799b52"; + }) + (fetchurl { + url = "https://mirror.bazel.build/github.com/bazelbuild/rules_sass/archive/8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz"; + sha256 = "d868ce50d592ef4aad7dec4dd32ae68d2151261913450fac8390b3fd474bb898"; + }) + (fetchurl { + url = "https://mirror.bazel.build/bazel_java_tools/releases/javac10/v3.1/java_tools_javac10_linux-v3.1.zip"; + sha256 = "a0cd51f9db1bf05a722ff7f5c60a07fa1c7d27428fff0815c342d32aa6c53576"; + }) + (fetchurl { + url = "https://mirror.bazel.build/bazel_java_tools/releases/javac10/v3.1/java_tools_javac10_darwin-v3.1.zip"; + sha256 = "c646aad8808b8ec5844d6a80a1287fc8e13203375fe40d6af4819eff48b9bbaf"; + }) + (fetchurl { + url = "https://mirror.bazel.build/bazel_coverage_output_generator/releases/coverage_output_generator-v1.0.zip"; + sha256 = "cc470e529fafb6165b5be3929ff2d99b38429b386ac100878687416603a67889"; + }) + (fetchurl { + url = "https://github.com/bazelbuild/rules_nodejs/archive/0.16.2.zip"; + sha256 = "9b72bb0aea72d7cbcfc82a01b1e25bf3d85f791e790ddec16c65e2d906382ee0"; + }) + (fetchurl { + url = "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.2.tar.gz"; + sha256 = "04f85f2dd049e87805511e3babc5cea3f5e72332b1627e34f3a5461cc38e815f"; + }) + ]; + + distDir = runCommand "bazel-deps" {} '' + mkdir -p $out + for i in ${builtins.toString srcDeps}; do cp $i $out/$(stripHash $i); done + ''; + + defaultShellPath = lib.makeBinPath + # Keep this list conservative. For more exotic tools, prefer to use + # @rules_nixpkgs to pull in tools from the nix repository. Example: + # + # WORKSPACE: + # + # nixpkgs_git_repository( + # name = "nixpkgs", + # revision = "def5124ec8367efdba95a99523dd06d918cb0ae8", + # ) + # + # # This defines an external Bazel workspace. + # nixpkgs_package( + # name = "bison", + # repositories = { "nixpkgs": "@nixpkgs//:default.nix" }, + # ) + # + # some/BUILD.bazel: + # + # genrule( + # ... + # cmd = "$(location @bison//:bin/bison) -other -args", + # tools = [ + # ... + # "@bison//:bin/bison", + # ], + # ) + # + [ bash coreutils findutils gawk gnugrep gnutar gnused gzip which unzip ]; + + # Java toolchain used for the build and tests + javaToolchain = "@bazel_tools//tools/jdk:toolchain_host${buildJdkName}"; + +in +stdenv.mkDerivation rec { + + version = "0.26.0"; + + meta = with lib; { + homepage = "https://github.com/bazelbuild/bazel/"; + description = "Build tool that builds code quickly and reliably"; + license = licenses.asl20; + platforms = platforms.linux ++ platforms.darwin; + }; + + # Additional tests that check bazel’s functionality. Execute + # + # nix-build . -A bazel.tests + # + # in the nixpkgs checkout root to exercise them locally. + passthru.tests = { + pythonBinPath = callPackage ./python-bin-path-test.nix {}; + bashTools = callPackage ./bash-tools-test.nix {}; + }; + + name = "bazel-${version}"; + + src = fetchurl { + url = "https://github.com/bazelbuild/bazel/releases/download/${version}/${name}-dist.zip"; + sha256 = "d26dadf62959255d58e523da3448a6222af768fe1224e321b120c1d5bbe4b4f2"; + }; + + # Necessary for the tests to pass on Darwin with sandbox enabled. + # Bazel starts a local server and needs to bind a local address. + __darwinAllowLocalNetworking = true; + + sourceRoot = "."; + + patches = [ + ./glibc.patch + ./python-stub-path-fix.patch + ] ++ lib.optional enableNixHacks ../nix-hacks.patch; + + # Bazel expects several utils to be available in Bash even without PATH. Hence this hack. + + customBash = writeCBin "bash" '' + #include + #include + #include + #include + + extern char **environ; + + int main(int argc, char *argv[]) { + char *path = getenv("PATH"); + char *pathToAppend = "${defaultShellPath}"; + char *newPath; + if (path != NULL) { + int length = strlen(path) + 1 + strlen(pathToAppend) + 1; + newPath = malloc(length * sizeof(char)); + snprintf(newPath, length, "%s:%s", path, pathToAppend); + } else { + newPath = pathToAppend; + } + setenv("PATH", newPath, 1); + execve("${bash}/bin/bash", argv, environ); + return 0; + } + ''; + + postPatch = let + + darwinPatches = '' + # Disable Bazel's Xcode toolchain detection which would configure compilers + # and linkers from Xcode instead of from PATH + export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 + + # Explicitly configure gcov since we don't have it on Darwin, so autodetection fails + export GCOV=${coreutils}/bin/false + + # Framework search paths aren't added by bintools hook + # https://github.com/NixOS/nixpkgs/pull/41914 + export NIX_LDFLAGS="$NIX_LDFLAGS -F${CoreFoundation}/Library/Frameworks -F${CoreServices}/Library/Frameworks -F${Foundation}/Library/Frameworks" + + # libcxx includes aren't added by libcxx hook + # https://github.com/NixOS/nixpkgs/pull/41589 + export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -isystem ${libcxx}/include/c++/v1" + + # don't use system installed Xcode to run clang, use Nix clang instead + sed -i -e "s;/usr/bin/xcrun clang;${stdenv.cc}/bin/clang $NIX_CFLAGS_COMPILE $NIX_LDFLAGS -framework CoreFoundation;g" \ + scripts/bootstrap/compile.sh \ + src/tools/xcode/realpath/BUILD \ + src/tools/xcode/stdredirect/BUILD \ + tools/osx/BUILD + + # nixpkgs's libSystem cannot use pthread headers directly, must import GCD headers instead + sed -i -e "/#include /i #include " src/main/cpp/blaze_util_darwin.cc + + # clang installed from Xcode has a compatibility wrapper that forwards + # invocations of gcc to clang, but vanilla clang doesn't + sed -i -e 's;_find_generic(repository_ctx, "gcc", "CC", overriden_tools);_find_generic(repository_ctx, "clang", "CC", overriden_tools);g' tools/cpp/unix_cc_configure.bzl + + sed -i -e 's;/usr/bin/libtool;${cctools}/bin/libtool;g' tools/cpp/unix_cc_configure.bzl + wrappers=( tools/cpp/osx_cc_wrapper.sh tools/cpp/osx_cc_wrapper.sh.tpl ) + for wrapper in "''${wrappers[@]}"; do + sed -i -e "s,/usr/bin/install_name_tool,${cctools}/bin/install_name_tool,g" $wrapper + done + ''; + + genericPatches = '' + # Substitute python's stub shebang to plain python path. (see TODO add pr URL) + # See also `postFixup` where python is added to $out/nix-support + substituteInPlace src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt\ + --replace "/usr/bin/env python" "${python}/bin/python" \ + --replace "NIX_STORE_PYTHON_PATH" "${python}/bin/python" \ + + # md5sum is part of coreutils + sed -i 's|/sbin/md5|md5sum|' \ + src/BUILD + + # substituteInPlace is rather slow, so prefilter the files with grep + grep -rlZ /bin src/main/java/com/google/devtools | while IFS="" read -r -d "" path; do + # If you add more replacements here, you must change the grep above! + # Only files containing /bin are taken into account. + substituteInPlace "$path" \ + --replace /bin/bash ${customBash}/bin/bash \ + --replace /usr/bin/env ${coreutils}/bin/env \ + --replace /bin/true ${coreutils}/bin/true + done + + # Fixup scripts that generate scripts. Not fixed up by patchShebangs below. + substituteInPlace scripts/bootstrap/compile.sh \ + --replace /bin/bash ${customBash}/bin/bash + + # add nix environment vars to .bazelrc + cat >> .bazelrc <> runfiles.bash.tmp + cat tools/bash/runfiles/runfiles.bash >> runfiles.bash.tmp + mv runfiles.bash.tmp tools/bash/runfiles/runfiles.bash + + patchShebangs . + ''; + in lib.optionalString stdenv.hostPlatform.isDarwin darwinPatches + + genericPatches; + + buildInputs = [ + buildJdk + ]; + + # when a command can’t be found in a bazel build, you might also + # need to add it to `defaultShellPath`. + nativeBuildInputs = [ + zip + python + unzip + makeWrapper + which + customBash + ] ++ lib.optionals (stdenv.isDarwin) [ cctools libcxx CoreFoundation CoreServices Foundation ]; + + # Bazel makes extensive use of symlinks in the WORKSPACE. + # This causes problems with infinite symlinks if the build output is in the same location as the + # Bazel WORKSPACE. This is why before executing the build, the source code is moved into a + # subdirectory. + # Failing to do this causes "infinite symlink expansion detected" + preBuildPhases = ["preBuildPhase"]; + preBuildPhase = '' + mkdir bazel_src + shopt -s dotglob extglob + mv !(bazel_src) bazel_src + ''; + + buildPhase = '' + # Increasing memory during compilation might be necessary. + # export BAZEL_JAVAC_OPTS="-J-Xmx2g -J-Xms200m" + ./bazel_src/compile.sh + ./bazel_src/scripts/generate_bash_completion.sh \ + --bazel=./bazel_src/output/bazel \ + --output=./bazel_src/output/bazel-complete.bash \ + --prepend=./bazel_src/scripts/bazel-complete-header.bash \ + --prepend=./bazel_src/scripts/bazel-complete-template.bash + ''; + + installPhase = '' + mkdir -p $out/bin + + # official wrapper scripts that searches for $WORKSPACE_ROOT/tools/bazel + # if it can’t find something in tools, it calls $out/bin/bazel-real + cp ./bazel_src/scripts/packages/bazel.sh $out/bin/bazel + mv ./bazel_src/output/bazel $out/bin/bazel-real + + wrapProgram "$out/bin/bazel" --add-flags --server_javabase="${runJdk}" + + # shell completion files + mkdir -p $out/share/bash-completion/completions $out/share/zsh/site-functions + mv ./bazel_src/output/bazel-complete.bash $out/share/bash-completion/completions/bazel + cp ./bazel_src/scripts/zsh_completion/_bazel $out/share/zsh/site-functions/ + ''; + + # Temporarily disabling for now. A new approach is needed for this derivation as Bazel + # accesses the internet during the tests which fails in a sandbox. + doInstallCheck = false; + installCheckPhase = '' + export TEST_TMPDIR=$(pwd) + + hello_test () { + $out/bin/bazel test \ + --test_output=errors \ + --java_toolchain='${javaToolchain}' \ + examples/cpp:hello-success_test \ + examples/java-native/src/test/java/com/example/myproject:hello + } + + cd ./bazel_src + + # test whether $WORKSPACE_ROOT/tools/bazel works + + mkdir -p tools + cat > tools/bazel <<"EOF" + #!${runtimeShell} -e + exit 1 + EOF + chmod +x tools/bazel + + # first call should fail if tools/bazel is used + ! hello_test + + cat > tools/bazel <<"EOF" + #!${runtimeShell} -e + exec "$BAZEL_REAL" "$@" + EOF + + # second call succeeds because it defers to $out/bin/bazel-real + hello_test + ''; + + # Save paths to hardcoded dependencies so Nix can detect them. + postFixup = '' + mkdir -p $out/nix-support + echo "${customBash} ${defaultShellPath}" >> $out/nix-support/depends + # The templates get tar’d up into a .jar, + # so nix can’t detect python is needed in the runtime closure + echo "${python}" >> $out/nix-support/depends + ''; + + dontStrip = true; + dontPatchELF = true; +} diff --git a/pkgs/development/tools/build-managers/bazel/bazel_0_26/glibc.patch b/pkgs/development/tools/build-managers/bazel/bazel_0_26/glibc.patch new file mode 100644 index 000000000000..c4de48068f11 --- /dev/null +++ b/pkgs/development/tools/build-managers/bazel/bazel_0_26/glibc.patch @@ -0,0 +1,78 @@ +From https://github.com/grpc/grpc/commit/57586a1ca7f17b1916aed3dea4ff8de872dbf853 +From: Benjamin Peterson +Date: Fri, 3 May 2019 08:11:00 -0700 +Subject: [PATCH] Rename gettid() functions. + +glibc 2.30 will declare its own gettid; see https://sourceware.org/git/?p=glibc.git;a=commit;h=1d0fc213824eaa2a8f8c4385daaa698ee8fb7c92. Rename the grpc versions to avoid naming conflicts. +--- + src/core/lib/gpr/log_linux.cc | 6 ++---- + src/core/lib/gpr/log_posix.cc | 4 ++-- + src/core/lib/iomgr/ev_epollex_linux.cc | 4 ++-- + 3 files changed, 6 insertions(+), 8 deletions(-) + +diff --git a/third_party/grpc/src/core/lib/gpr/log_linux.cc b/src/core/lib/gpr/log_linux.cc +index 81026e5689b..8b597b4cf2f 100644 +--- a/third_party/grpc/src/core/lib/gpr/log_linux.cc ++++ b/third_party/grpc/src/core/lib/gpr/log_linux.cc +@@ -40,7 +40,7 @@ + #include + #include + +-static long gettid(void) { return syscall(__NR_gettid); } ++static long sys_gettid(void) { return syscall(__NR_gettid); } + + void gpr_log(const char* file, int line, gpr_log_severity severity, + const char* format, ...) { +@@ -70,7 +70,7 @@ void gpr_default_log(gpr_log_func_args* args) { + gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME); + struct tm tm; + static __thread long tid = 0; +- if (tid == 0) tid = gettid(); ++ if (tid == 0) tid = sys_gettid(); + + timer = static_cast(now.tv_sec); + final_slash = strrchr(args->file, '/'); +diff --git a/third_party/grpc/src/core/lib/gpr/log_posix.cc b/src/core/lib/gpr/log_posix.cc +index b6edc14ab6b..2f7c6ce3760 100644 +--- a/third_party/grpc/src/core/lib/gpr/log_posix.cc ++++ b/third_party/grpc/src/core/lib/gpr/log_posix.cc +@@ -31,7 +31,7 @@ + #include + #include + +-static intptr_t gettid(void) { return (intptr_t)pthread_self(); } ++static intptr_t sys_gettid(void) { return (intptr_t)pthread_self(); } + + void gpr_log(const char* file, int line, gpr_log_severity severity, + const char* format, ...) { +@@ -86,7 +86,7 @@ void gpr_default_log(gpr_log_func_args* args) { + char* prefix; + gpr_asprintf(&prefix, "%s%s.%09d %7" PRIdPTR " %s:%d]", + gpr_log_severity_string(args->severity), time_buffer, +- (int)(now.tv_nsec), gettid(), display_file, args->line); ++ (int)(now.tv_nsec), sys_gettid(), display_file, args->line); + + fprintf(stderr, "%-70s %s\n", prefix, args->message); + gpr_free(prefix); +diff --git a/third_party/grpc/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc +index c2d80c08ddb..4a83cb6c215 100644 +--- a/third_party/grpc/src/core/lib/iomgr/ev_epollex_linux.cc ++++ b/third_party/grpc/src/core/lib/iomgr/ev_epollex_linux.cc +@@ -1077,7 +1077,7 @@ static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, + } + + #ifndef NDEBUG +-static long gettid(void) { return syscall(__NR_gettid); } ++static long sys_gettid(void) { return syscall(__NR_gettid); } + #endif + + /* pollset->mu lock must be held by the caller before calling this. +@@ -1097,7 +1097,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset, + #define WORKER_PTR (&worker) + #endif + #ifndef NDEBUG +- WORKER_PTR->originator = gettid(); ++ WORKER_PTR->originator = sys_gettid(); + #endif + if (GRPC_TRACE_FLAG_ENABLED(grpc_polling_trace)) { + gpr_log(GPR_INFO, diff --git a/pkgs/development/tools/build-managers/bazel/bazel_0_26/python-stub-path-fix.patch b/pkgs/development/tools/build-managers/bazel/bazel_0_26/python-stub-path-fix.patch new file mode 100644 index 000000000000..cbc4192d2d9b --- /dev/null +++ b/pkgs/development/tools/build-managers/bazel/bazel_0_26/python-stub-path-fix.patch @@ -0,0 +1,13 @@ +diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt +index dac21c9a83..69b11c283f 100644 +--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt ++++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt +@@ -67,7 +67,7 @@ def FindPythonBinary(module_space): + return os.path.join(module_space, PYTHON_BINARY) + else: + # Case 4: Path has to be looked up in the search path. +- return SearchPath(PYTHON_BINARY) ++ return "NIX_STORE_PYTHON_PATH" + + def CreatePythonPathEntries(python_imports, module_space): + parts = python_imports.split(':'); diff --git a/pkgs/development/tools/build-managers/bazel/bazel_0_29/default.nix b/pkgs/development/tools/build-managers/bazel/bazel_0_29/default.nix new file mode 100644 index 000000000000..68adddebb0ac --- /dev/null +++ b/pkgs/development/tools/build-managers/bazel/bazel_0_29/default.nix @@ -0,0 +1,559 @@ +{ stdenv, callPackage, lib, fetchurl, fetchFromGitHub +, runCommand, runCommandCC, makeWrapper, recurseIntoAttrs +# this package (through the fixpoint glass) +, bazel_1 +, lr, xe, zip, unzip, bash, writeCBin, coreutils +, which, gawk, gnused, gnutar, gnugrep, gzip, findutils +# updater +, python27, python3, writeScript +# Apple dependencies +, cctools, libcxx, CoreFoundation, CoreServices, Foundation +# Allow to independently override the jdks used to build and run respectively +, buildJdk, runJdk +, buildJdkName +, runtimeShell +# Downstream packages for tests +, bazel-watcher +# Always assume all markers valid (this is needed because we remove markers; they are non-deterministic). +# Also, don't clean up environment variables (so that NIX_ environment variables are passed to compilers). +, enableNixHacks ? false +, gcc-unwrapped +, autoPatchelfHook +, file +, substituteAll +, writeTextFile +}: + +let + version = "0.29.1"; + + src = fetchurl { + url = "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-dist.zip"; + sha256 = "1rcd6xy61n07n7m6dgcw23275r8z3gkwmqdkd48nwrq8yb7m4al7"; + }; + + # Update with `eval $(nix-build -A bazel.updater)`, + # then add new dependencies from the dict in ./src-deps.json as required. + srcDeps = lib.attrsets.attrValues srcDepsSet; + srcDepsSet = + let + srcs = (builtins.fromJSON (builtins.readFile ./src-deps.json)); + toFetchurl = d: lib.attrsets.nameValuePair d.name (fetchurl { + urls = d.urls; + sha256 = d.sha256; + }); + in builtins.listToAttrs (map toFetchurl [ + srcs.desugar_jdk_libs + srcs.io_bazel_skydoc + srcs.bazel_skylib + srcs.io_bazel_rules_sass + srcs.platforms + (if stdenv.hostPlatform.isDarwin + then srcs."java_tools_javac11_darwin-v4.0.zip" + else srcs."java_tools_javac11_linux-v4.0.zip") + srcs."coverage_output_generator-v1.0.zip" + srcs.build_bazel_rules_nodejs + srcs."android_tools_pkg-0.8.tar.gz" + srcs."0.27.1.tar.gz" + srcs.rules_pkg + srcs.rules_cc + srcs.rules_java + srcs.rules_proto + ]); + + distDir = runCommand "bazel-deps" {} '' + mkdir -p $out + for i in ${builtins.toString srcDeps}; do cp $i $out/$(stripHash $i); done + ''; + + defaultShellPath = lib.makeBinPath + # Keep this list conservative. For more exotic tools, prefer to use + # @rules_nixpkgs to pull in tools from the nix repository. Example: + # + # WORKSPACE: + # + # nixpkgs_git_repository( + # name = "nixpkgs", + # revision = "def5124ec8367efdba95a99523dd06d918cb0ae8", + # ) + # + # # This defines an external Bazel workspace. + # nixpkgs_package( + # name = "bison", + # repositories = { "nixpkgs": "@nixpkgs//:default.nix" }, + # ) + # + # some/BUILD.bazel: + # + # genrule( + # ... + # cmd = "$(location @bison//:bin/bison) -other -args", + # tools = [ + # ... + # "@bison//:bin/bison", + # ], + # ) + # + [ bash coreutils findutils gawk gnugrep gnutar gnused gzip which unzip file zip ]; + + # Java toolchain used for the build and tests + javaToolchain = "@bazel_tools//tools/jdk:toolchain_host${buildJdkName}"; + + platforms = lib.platforms.linux ++ lib.platforms.darwin; + + # This repository is fetched by bazel at runtime + # however it contains prebuilt java binaries, with wrong interpreter + # and libraries path. + # We prefetch it, patch it, and override it in a global bazelrc. + system = if stdenv.hostPlatform.isDarwin then "darwin" else "linux"; + + remote_java_tools = stdenv.mkDerivation { + name = "remote_java_tools_${system}"; + + src = srcDepsSet."java_tools_javac11_${system}-v4.0.zip"; + + nativeBuildInputs = [ autoPatchelfHook unzip ]; + buildInputs = [ gcc-unwrapped ]; + + sourceRoot = "."; + + buildPhase = '' + mkdir $out; + ''; + + installPhase = '' + cp -Ra * $out/ + touch $out/WORKSPACE + ''; + }; + + bazelRC = writeTextFile { + name = "bazel-rc"; + text = '' + build --override_repository=${remote_java_tools.name}=${remote_java_tools} + build --distdir=${distDir} + startup --server_javabase=${runJdk} + + # load default location for the system wide configuration + try-import /etc/bazel.bazelrc + ''; + }; + +in +stdenv.mkDerivation rec { + pname = "bazel"; + inherit version; + + meta = with lib; { + homepage = "https://github.com/bazelbuild/bazel/"; + description = "Build tool that builds code quickly and reliably"; + license = licenses.asl20; + maintainers = [ maintainers.mboes ]; + inherit platforms; + }; + + inherit src; + sourceRoot = "."; + + patches = [ + # On Darwin, the last argument to gcc is coming up as an empty string. i.e: '' + # This is breaking the build of any C target. This patch removes the last + # argument if it's found to be an empty string. + ../trim-last-argument-to-gcc-if-empty.patch + ./glibc.patch + + # --experimental_strict_action_env (which may one day become the default + # see bazelbuild/bazel#2574) hardcodes the default + # action environment to a non hermetic value (e.g. "/usr/local/bin"). + # This is non hermetic on non-nixos systems. On NixOS, bazel cannot find the required binaries. + # So we are replacing this bazel paths by defaultShellPath, + # improving hermeticity and making it work in nixos. + (substituteAll { + src = ../strict_action_env.patch; + strictActionEnvPatch = defaultShellPath; + }) + + # bazel reads its system bazelrc in /etc + # override this path to a builtin one + (substituteAll { + src = ../bazel_rc.patch; + bazelSystemBazelRCPath = bazelRC; + }) + ] ++ lib.optional enableNixHacks ../nix-hacks.patch; + + + # Additional tests that check bazel’s functionality. Execute + # + # nix-build . -A bazel.tests + # + # in the nixpkgs checkout root to exercise them locally. + passthru.tests = + let + runLocal = name: attrs: script: + let + attrs' = removeAttrs attrs [ "buildInputs" ]; + buildInputs = [ python3 ] ++ (attrs.buildInputs or []); + in + runCommandCC name ({ + inherit buildInputs; + preferLocalBuild = true; + meta.platforms = platforms; + } // attrs') script; + + # bazel wants to extract itself into $install_dir/install every time it runs, + # so let’s do that only once. + extracted = bazelPkg: + let install_dir = + # `install_base` field printed by `bazel info`, minus the hash. + # yes, this path is kinda magic. Sorry. + "$HOME/.cache/bazel/_bazel_nixbld"; + in runLocal "bazel-extracted-homedir" { passthru.install_dir = install_dir; } '' + export HOME=$(mktemp -d) + touch WORKSPACE # yeah, everything sucks + install_base="$(${bazelPkg}/bin/bazel info | grep install_base)" + # assert it’s actually below install_dir + [[ "$install_base" =~ ${install_dir} ]] \ + || (echo "oh no! $install_base but we are \ + trying to copy ${install_dir} to $out instead!"; exit 1) + cp -R ${install_dir} $out + ''; + + bazelTest = { name, bazelScript, workspaceDir, bazelPkg, buildInputs ? [] }: + let + be = extracted bazelPkg; + in runLocal name { inherit buildInputs; } ( + # skip extraction caching on Darwin, because nobody knows how Darwin works + (lib.optionalString (!stdenv.hostPlatform.isDarwin) '' + # set up home with pre-unpacked bazel + export HOME=$(mktemp -d) + mkdir -p ${be.install_dir} + cp -R ${be}/install ${be.install_dir} + + # https://stackoverflow.com/questions/47775668/bazel-how-to-skip-corrupt-installation-on-centos6 + # Bazel checks whether the mtime of the install dir files + # is >9 years in the future, otherwise it extracts itself again. + # see PosixFileMTime::IsUntampered in src/main/cpp/util + # What the hell bazel. + ${lr}/bin/lr -0 -U ${be.install_dir} | ${xe}/bin/xe -N0 -0 touch --date="9 years 6 months" {} + '') + + + '' + # Note https://github.com/bazelbuild/bazel/issues/5763#issuecomment-456374609 + # about why to create a subdir for the workspace. + cp -r ${workspaceDir} wd && chmod u+w wd && cd wd + + ${bazelScript} + + touch $out + ''); + + bazelWithNixHacks = bazel_1.override { enableNixHacks = true; }; + + bazel-examples = fetchFromGitHub { + owner = "bazelbuild"; + repo = "examples"; + rev = "5d8c8961a2516ebf875787df35e98cadd08d43dc"; + sha256 = "03c1bwlq5bs3hg96v4g4pg2vqwhqq6w538h66rcpw02f83yy7fs8"; + }; + + in (if !stdenv.hostPlatform.isDarwin then { + # `extracted` doesn’t work on darwin + shebang = callPackage ./shebang-test.nix { inherit runLocal extracted bazelTest distDir; }; + } else {}) // { + bashTools = callPackage ./bash-tools-test.nix { inherit runLocal bazelTest distDir; }; + cpp = callPackage ./cpp-test.nix { inherit runLocal bazelTest bazel-examples distDir; }; + java = callPackage ./java-test.nix { inherit runLocal bazelTest bazel-examples distDir; }; + protobuf = callPackage ./protobuf-test.nix { inherit runLocal bazelTest distDir; }; + pythonBinPath = callPackage ./python-bin-path-test.nix { inherit runLocal bazelTest distDir; }; + + bashToolsWithNixHacks = callPackage ./bash-tools-test.nix { inherit runLocal bazelTest distDir; bazel = bazelWithNixHacks; }; + + cppWithNixHacks = callPackage ./cpp-test.nix { inherit runLocal bazelTest bazel-examples distDir; bazel = bazelWithNixHacks; }; + javaWithNixHacks = callPackage ./java-test.nix { inherit runLocal bazelTest bazel-examples distDir; bazel = bazelWithNixHacks; }; + protobufWithNixHacks = callPackage ./protobuf-test.nix { inherit runLocal bazelTest distDir; bazel = bazelWithNixHacks; }; + pythonBinPathWithNixHacks = callPackage ./python-bin-path-test.nix { inherit runLocal bazelTest distDir; bazel = bazelWithNixHacks; }; + + # downstream packages using buildBazelPackage + # fixed-output hashes of the fetch phase need to be spot-checked manually + downstream = recurseIntoAttrs ({ + inherit bazel-watcher; + } + # dm-sonnet is only packaged for linux + // (lib.optionalAttrs stdenv.isLinux { + # TODO(timokau) dm-sonnet is broken currently + # dm-sonnet-linux = python3.pkgs.dm-sonnet; + })); + }; + + # update the list of workspace dependencies + passthru.updater = writeScript "update-bazel-deps.sh" '' + #!${runtimeShell} + cat ${runCommand "bazel-deps.json" {} '' + ${unzip}/bin/unzip ${src} WORKSPACE + ${python3}/bin/python3 ${../update-srcDeps.py} ./WORKSPACE > $out + ''} > ${builtins.toString ./src-deps.json} + ''; + + # Necessary for the tests to pass on Darwin with sandbox enabled. + # Bazel starts a local server and needs to bind a local address. + __darwinAllowLocalNetworking = true; + + # Bazel expects several utils to be available in Bash even without PATH. Hence this hack. + customBash = writeCBin "bash" '' + #include + #include + #include + #include + + extern char **environ; + + int main(int argc, char *argv[]) { + char *path = getenv("PATH"); + char *pathToAppend = "${defaultShellPath}"; + char *newPath; + if (path != NULL) { + int length = strlen(path) + 1 + strlen(pathToAppend) + 1; + newPath = malloc(length * sizeof(char)); + snprintf(newPath, length, "%s:%s", path, pathToAppend); + } else { + newPath = pathToAppend; + } + setenv("PATH", newPath, 1); + execve("${bash}/bin/bash", argv, environ); + return 0; + } + ''; + + postPatch = let + + darwinPatches = '' + bazelLinkFlags () { + eval set -- "$NIX_LDFLAGS" + local flag + for flag in "$@"; do + printf ' -Wl,%s' "$flag" + done + } + + # Disable Bazel's Xcode toolchain detection which would configure compilers + # and linkers from Xcode instead of from PATH + export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 + + # Explicitly configure gcov since we don't have it on Darwin, so autodetection fails + export GCOV=${coreutils}/bin/false + + # Framework search paths aren't added by bintools hook + # https://github.com/NixOS/nixpkgs/pull/41914 + export NIX_LDFLAGS+=" -F${CoreFoundation}/Library/Frameworks -F${CoreServices}/Library/Frameworks -F${Foundation}/Library/Frameworks" + + # libcxx includes aren't added by libcxx hook + # https://github.com/NixOS/nixpkgs/pull/41589 + export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -isystem ${libcxx}/include/c++/v1" + + # don't use system installed Xcode to run clang, use Nix clang instead + sed -i -E "s;/usr/bin/xcrun (--sdk macosx )?clang;${stdenv.cc}/bin/clang $NIX_CFLAGS_COMPILE $(bazelLinkFlags) -framework CoreFoundation;g" \ + scripts/bootstrap/compile.sh \ + src/tools/xcode/realpath/BUILD \ + src/tools/xcode/stdredirect/BUILD \ + tools/osx/BUILD + + # nixpkgs's libSystem cannot use pthread headers directly, must import GCD headers instead + sed -i -e "/#include /i #include " src/main/cpp/blaze_util_darwin.cc + + # clang installed from Xcode has a compatibility wrapper that forwards + # invocations of gcc to clang, but vanilla clang doesn't + sed -i -e 's;_find_generic(repository_ctx, "gcc", "CC", overriden_tools);_find_generic(repository_ctx, "clang", "CC", overriden_tools);g' tools/cpp/unix_cc_configure.bzl + + sed -i -e 's;/usr/bin/libtool;${cctools}/bin/libtool;g' tools/cpp/unix_cc_configure.bzl + wrappers=( tools/cpp/osx_cc_wrapper.sh tools/cpp/osx_cc_wrapper.sh.tpl ) + for wrapper in "''${wrappers[@]}"; do + sed -i -e "s,/usr/bin/install_name_tool,${cctools}/bin/install_name_tool,g" $wrapper + done + ''; + + genericPatches = '' + # Substitute j2objc and objc wrapper's python shebang to plain python path. + # These scripts explicitly depend on Python 2.7, hence we use python27. + # See also `postFixup` where python27 is added to $out/nix-support + substituteInPlace tools/j2objc/j2objc_header_map.py --replace "$!/usr/bin/python2.7" "#!${python27}/bin/python" + substituteInPlace tools/j2objc/j2objc_wrapper.py --replace "$!/usr/bin/python2.7" "#!${python27}/bin/python" + substituteInPlace tools/objc/j2objc_dead_code_pruner.py --replace "$!/usr/bin/python2.7" "#!${python27}/bin/python" + + # md5sum is part of coreutils + sed -i 's|/sbin/md5|md5sum|' \ + src/BUILD + + # substituteInPlace is rather slow, so prefilter the files with grep + grep -rlZ /bin src/main/java/com/google/devtools | while IFS="" read -r -d "" path; do + # If you add more replacements here, you must change the grep above! + # Only files containing /bin are taken into account. + # We default to python3 where possible. See also `postFixup` where + # python3 is added to $out/nix-support + substituteInPlace "$path" \ + --replace /bin/bash ${customBash}/bin/bash \ + --replace "/usr/bin/env bash" ${customBash}/bin/bash \ + --replace "/usr/bin/env python" ${python3}/bin/python \ + --replace /usr/bin/env ${coreutils}/bin/env \ + --replace /bin/true ${coreutils}/bin/true + done + + # bazel test runner include references to /bin/bash + substituteInPlace tools/build_rules/test_rules.bzl \ + --replace /bin/bash ${customBash}/bin/bash + + for i in $(find tools/cpp/ -type f) + do + substituteInPlace $i \ + --replace /bin/bash ${customBash}/bin/bash + done + + # Fixup scripts that generate scripts. Not fixed up by patchShebangs below. + substituteInPlace scripts/bootstrap/compile.sh \ + --replace /bin/bash ${customBash}/bin/bash + + # add nix environment vars to .bazelrc + cat >> .bazelrc <> runfiles.bash.tmp + cat tools/bash/runfiles/runfiles.bash >> runfiles.bash.tmp + mv runfiles.bash.tmp tools/bash/runfiles/runfiles.bash + + patchShebangs . + ''; + in lib.optionalString stdenv.hostPlatform.isDarwin darwinPatches + + genericPatches; + + buildInputs = [ + buildJdk + python3 + ]; + + # when a command can’t be found in a bazel build, you might also + # need to add it to `defaultShellPath`. + nativeBuildInputs = [ + zip + python3 + unzip + makeWrapper + which + customBash + ] ++ lib.optionals (stdenv.isDarwin) [ cctools libcxx CoreFoundation CoreServices Foundation ]; + + # Bazel makes extensive use of symlinks in the WORKSPACE. + # This causes problems with infinite symlinks if the build output is in the same location as the + # Bazel WORKSPACE. This is why before executing the build, the source code is moved into a + # subdirectory. + # Failing to do this causes "infinite symlink expansion detected" + preBuildPhases = ["preBuildPhase"]; + preBuildPhase = '' + mkdir bazel_src + shopt -s dotglob extglob + mv !(bazel_src) bazel_src + ''; + + buildPhase = '' + # Increasing memory during compilation might be necessary. + # export BAZEL_JAVAC_OPTS="-J-Xmx2g -J-Xms200m" + ./bazel_src/compile.sh + ./bazel_src/scripts/generate_bash_completion.sh \ + --bazel=./bazel_src/output/bazel \ + --output=./bazel_src/output/bazel-complete.bash \ + --prepend=./bazel_src/scripts/bazel-complete-header.bash \ + --prepend=./bazel_src/scripts/bazel-complete-template.bash + ''; + + installPhase = '' + mkdir -p $out/bin + + # official wrapper scripts that searches for $WORKSPACE_ROOT/tools/bazel + # if it can’t find something in tools, it calls $out/bin/bazel-real + cp ./bazel_src/scripts/packages/bazel.sh $out/bin/bazel + mv ./bazel_src/output/bazel $out/bin/bazel-real + + # shell completion files + mkdir -p $out/share/bash-completion/completions $out/share/zsh/site-functions + mv ./bazel_src/output/bazel-complete.bash $out/share/bash-completion/completions/bazel + cp ./bazel_src/scripts/zsh_completion/_bazel $out/share/zsh/site-functions/ + ''; + + doInstallCheck = true; + installCheckPhase = '' + export TEST_TMPDIR=$(pwd) + + hello_test () { + $out/bin/bazel test --distdir=${distDir} \ + --test_output=errors \ + --java_toolchain='${javaToolchain}' \ + examples/cpp:hello-success_test \ + examples/java-native/src/test/java/com/example/myproject:hello + } + + cd ./bazel_src + + # test whether $WORKSPACE_ROOT/tools/bazel works + + mkdir -p tools + cat > tools/bazel <<"EOF" + #!${runtimeShell} -e + exit 1 + EOF + chmod +x tools/bazel + + # first call should fail if tools/bazel is used + ! hello_test + + cat > tools/bazel <<"EOF" + #!${runtimeShell} -e + exec "$BAZEL_REAL" "$@" + EOF + + # second call succeeds because it defers to $out/bin/bazel-real + hello_test + ''; + + # Save paths to hardcoded dependencies so Nix can detect them. + postFixup = '' + mkdir -p $out/nix-support + echo "${customBash} ${defaultShellPath}" >> $out/nix-support/depends + # The templates get tar’d up into a .jar, + # so nix can’t detect python is needed in the runtime closure + # Some of the scripts explicitly depend on Python 2.7. Otherwise, we + # default to using python3. Therefore, both python27 and python3 are + # runtime dependencies. + echo "${python27}" >> $out/nix-support/depends + echo "${python3}" >> $out/nix-support/depends + '' + lib.optionalString stdenv.isDarwin '' + echo "${cctools}" >> $out/nix-support/depends + ''; + + dontStrip = true; + dontPatchELF = true; +} diff --git a/pkgs/development/tools/build-managers/bazel/bazel_0_29/glibc.patch b/pkgs/development/tools/build-managers/bazel/bazel_0_29/glibc.patch new file mode 100644 index 000000000000..c4de48068f11 --- /dev/null +++ b/pkgs/development/tools/build-managers/bazel/bazel_0_29/glibc.patch @@ -0,0 +1,78 @@ +From https://github.com/grpc/grpc/commit/57586a1ca7f17b1916aed3dea4ff8de872dbf853 +From: Benjamin Peterson +Date: Fri, 3 May 2019 08:11:00 -0700 +Subject: [PATCH] Rename gettid() functions. + +glibc 2.30 will declare its own gettid; see https://sourceware.org/git/?p=glibc.git;a=commit;h=1d0fc213824eaa2a8f8c4385daaa698ee8fb7c92. Rename the grpc versions to avoid naming conflicts. +--- + src/core/lib/gpr/log_linux.cc | 6 ++---- + src/core/lib/gpr/log_posix.cc | 4 ++-- + src/core/lib/iomgr/ev_epollex_linux.cc | 4 ++-- + 3 files changed, 6 insertions(+), 8 deletions(-) + +diff --git a/third_party/grpc/src/core/lib/gpr/log_linux.cc b/src/core/lib/gpr/log_linux.cc +index 81026e5689b..8b597b4cf2f 100644 +--- a/third_party/grpc/src/core/lib/gpr/log_linux.cc ++++ b/third_party/grpc/src/core/lib/gpr/log_linux.cc +@@ -40,7 +40,7 @@ + #include + #include + +-static long gettid(void) { return syscall(__NR_gettid); } ++static long sys_gettid(void) { return syscall(__NR_gettid); } + + void gpr_log(const char* file, int line, gpr_log_severity severity, + const char* format, ...) { +@@ -70,7 +70,7 @@ void gpr_default_log(gpr_log_func_args* args) { + gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME); + struct tm tm; + static __thread long tid = 0; +- if (tid == 0) tid = gettid(); ++ if (tid == 0) tid = sys_gettid(); + + timer = static_cast(now.tv_sec); + final_slash = strrchr(args->file, '/'); +diff --git a/third_party/grpc/src/core/lib/gpr/log_posix.cc b/src/core/lib/gpr/log_posix.cc +index b6edc14ab6b..2f7c6ce3760 100644 +--- a/third_party/grpc/src/core/lib/gpr/log_posix.cc ++++ b/third_party/grpc/src/core/lib/gpr/log_posix.cc +@@ -31,7 +31,7 @@ + #include + #include + +-static intptr_t gettid(void) { return (intptr_t)pthread_self(); } ++static intptr_t sys_gettid(void) { return (intptr_t)pthread_self(); } + + void gpr_log(const char* file, int line, gpr_log_severity severity, + const char* format, ...) { +@@ -86,7 +86,7 @@ void gpr_default_log(gpr_log_func_args* args) { + char* prefix; + gpr_asprintf(&prefix, "%s%s.%09d %7" PRIdPTR " %s:%d]", + gpr_log_severity_string(args->severity), time_buffer, +- (int)(now.tv_nsec), gettid(), display_file, args->line); ++ (int)(now.tv_nsec), sys_gettid(), display_file, args->line); + + fprintf(stderr, "%-70s %s\n", prefix, args->message); + gpr_free(prefix); +diff --git a/third_party/grpc/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc +index c2d80c08ddb..4a83cb6c215 100644 +--- a/third_party/grpc/src/core/lib/iomgr/ev_epollex_linux.cc ++++ b/third_party/grpc/src/core/lib/iomgr/ev_epollex_linux.cc +@@ -1077,7 +1077,7 @@ static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, + } + + #ifndef NDEBUG +-static long gettid(void) { return syscall(__NR_gettid); } ++static long sys_gettid(void) { return syscall(__NR_gettid); } + #endif + + /* pollset->mu lock must be held by the caller before calling this. +@@ -1097,7 +1097,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset, + #define WORKER_PTR (&worker) + #endif + #ifndef NDEBUG +- WORKER_PTR->originator = gettid(); ++ WORKER_PTR->originator = sys_gettid(); + #endif + if (GRPC_TRACE_FLAG_ENABLED(grpc_polling_trace)) { + gpr_log(GPR_INFO, diff --git a/pkgs/development/tools/build-managers/bazel/bazel_0_29/src-deps.json b/pkgs/development/tools/build-managers/bazel/bazel_0_29/src-deps.json new file mode 100644 index 000000000000..f93794249ec7 --- /dev/null +++ b/pkgs/development/tools/build-managers/bazel/bazel_0_29/src-deps.json @@ -0,0 +1,506 @@ +{ + "0.16.2.zip": { + "name": "0.16.2.zip", + "sha256": "9b72bb0aea72d7cbcfc82a01b1e25bf3d85f791e790ddec16c65e2d906382ee0", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_nodejs/archive/0.16.2.zip", + "https://github.com/bazelbuild/rules_nodejs/archive/0.16.2.zip" + ] + }, + "0.27.1.tar.gz": { + "name": "0.27.1.tar.gz", + "sha256": "28cb3666da80fbc62d4c46814f5468dd5d0b59f9064c0b933eee3140d706d330", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/0.27.1.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/archive/0.27.1.tar.gz" + ] + }, + "0d5f3f2768c6ca2faca0079a997a97ce22997a0c.zip": { + "name": "0d5f3f2768c6ca2faca0079a997a97ce22997a0c.zip", + "sha256": "36fa66d4d49debd71d05fba55c1353b522e8caef4a20f8080a3d17cdda001d89", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_cc/archive/0d5f3f2768c6ca2faca0079a997a97ce22997a0c.zip", + "https://github.com/bazelbuild/rules_cc/archive/0d5f3f2768c6ca2faca0079a997a97ce22997a0c.zip" + ] + }, + "41c28e43dffbae39c52dd4b91932d1209e5a8893.tar.gz": { + "name": "41c28e43dffbae39c52dd4b91932d1209e5a8893.tar.gz", + "sha256": "fdc34621839104b57363a258eab9d821b02ff7837923cfe7fb6fd67182780829", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/skydoc/archive/41c28e43dffbae39c52dd4b91932d1209e5a8893.tar.gz", + "https://github.com/bazelbuild/skydoc/archive/41c28e43dffbae39c52dd4b91932d1209e5a8893.tar.gz" + ] + }, + "441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip": { + "name": "441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip", + "sha256": "a07fe5e75964361885db725039c2ba673f0ee0313d971ae4f50c9b18cd28b0b5", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip", + "https://github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip" + ] + }, + "7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip": { + "name": "7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip", + "sha256": "bc81f1ba47ef5cc68ad32225c3d0e70b8c6f6077663835438da8d5733f917598", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_java/archive/7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip", + "https://github.com/bazelbuild/rules_java/archive/7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip" + ] + }, + "8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz": { + "name": "8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz", + "sha256": "d868ce50d592ef4aad7dec4dd32ae68d2151261913450fac8390b3fd474bb898", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_sass/archive/8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz", + "https://github.com/bazelbuild/rules_sass/archive/8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz" + ] + }, + "android_tools_pkg-0.8.tar.gz": { + "name": "android_tools_pkg-0.8.tar.gz", + "sha256": "a9eac6e1b27d5549edaaa724b20eb1cdae6253b84f44d5744c30372bd523cfcd", + "urls": [ + "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.8.tar.gz" + ] + }, + "b0cc14be5da05168b01db282fe93bdf17aa2b9f4.tar.gz": { + "name": "b0cc14be5da05168b01db282fe93bdf17aa2b9f4.tar.gz", + "sha256": "88b0a90433866b44bb4450d4c30bc5738b8c4f9c9ba14e9661deb123f56a833d", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/b0cc14be5da05168b01db282fe93bdf17aa2b9f4.tar.gz", + "https://github.com/bazelbuild/rules_proto/archive/b0cc14be5da05168b01db282fe93bdf17aa2b9f4.tar.gz" + ] + }, + "bazel_j2objc": { + "name": "bazel_j2objc", + "sha256": "8d3403b5b7db57e347c943d214577f6879e5b175c2b59b7e075c0b6453330e9b", + "strip_prefix": "j2objc-2.5", + "urls": [ + "https://miirror.bazel.build/github.com/google/j2objc/releases/download/2.5/j2objc-2.5.zip", + "https://github.com/google/j2objc/releases/download/2.5/j2objc-2.5.zip" + ] + }, + "bazel_skylib": { + "name": "bazel_skylib", + "sha256": "ba5d15ca230efca96320085d8e4d58da826d1f81b444ef8afccd8b23e0799b52", + "strip_prefix": "bazel-skylib-f83cb8dd6f5658bc574ccd873e25197055265d1c", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/archive/f83cb8dd6f5658bc574ccd873e25197055265d1c.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/archive/f83cb8dd6f5658bc574ccd873e25197055265d1c.tar.gz" + ] + }, + "bazel_toolchains": { + "name": "bazel_toolchains", + "sha256": "28cb3666da80fbc62d4c46814f5468dd5d0b59f9064c0b933eee3140d706d330", + "strip_prefix": "bazel-toolchains-0.27.1", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/0.27.1.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/archive/0.27.1.tar.gz" + ] + }, + "build_bazel_rules_nodejs": { + "name": "build_bazel_rules_nodejs", + "sha256": "9b72bb0aea72d7cbcfc82a01b1e25bf3d85f791e790ddec16c65e2d906382ee0", + "strip_prefix": "rules_nodejs-0.16.2", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_nodejs/archive/0.16.2.zip", + "https://github.com/bazelbuild/rules_nodejs/archive/0.16.2.zip" + ] + }, + "com_google_googletest": { + "name": "com_google_googletest", + "sha256": "0fb00ff413f6b9b80ccee44a374ca7a18af7315aea72a43c62f2acd1ca74e9b5", + "strip_prefix": "googletest-f13bbe2992d188e834339abe6f715b2b2f840a77", + "urls": [ + "https://mirror.bazel.build/github.com/google/googletest/archive/f13bbe2992d188e834339abe6f715b2b2f840a77.tar.gz", + "https://github.com/google/googletest/archive/f13bbe2992d188e834339abe6f715b2b2f840a77.tar.gz" + ] + }, + "coverage_output_generator-v1.0.zip": { + "name": "coverage_output_generator-v1.0.zip", + "sha256": "cc470e529fafb6165b5be3929ff2d99b38429b386ac100878687416603a67889", + "urls": [ + "https://mirror.bazel.build/bazel_coverage_output_generator/releases/coverage_output_generator-v1.0.zip" + ] + }, + "desugar_jdk_libs": { + "name": "desugar_jdk_libs", + "sha256": "fe2e04f91ce8c59d49d91b8102edc6627c6fa2906c1b0e7346f01419ec4f419d", + "strip_prefix": "desugar_jdk_libs-e0b0291b2c51fbe5a7cfa14473a1ae850f94f021", + "urls": [ + "https://mirror.bazel.build/github.com/google/desugar_jdk_libs/archive/e0b0291b2c51fbe5a7cfa14473a1ae850f94f021.zip", + "https://github.com/google/desugar_jdk_libs/archive/e0b0291b2c51fbe5a7cfa14473a1ae850f94f021.zip" + ] + }, + "e0b0291b2c51fbe5a7cfa14473a1ae850f94f021.zip": { + "name": "e0b0291b2c51fbe5a7cfa14473a1ae850f94f021.zip", + "sha256": "fe2e04f91ce8c59d49d91b8102edc6627c6fa2906c1b0e7346f01419ec4f419d", + "urls": [ + "https://mirror.bazel.build/github.com/google/desugar_jdk_libs/archive/e0b0291b2c51fbe5a7cfa14473a1ae850f94f021.zip", + "https://github.com/google/desugar_jdk_libs/archive/e0b0291b2c51fbe5a7cfa14473a1ae850f94f021.zip" + ] + }, + "f83cb8dd6f5658bc574ccd873e25197055265d1c.tar.gz": { + "name": "f83cb8dd6f5658bc574ccd873e25197055265d1c.tar.gz", + "sha256": "ba5d15ca230efca96320085d8e4d58da826d1f81b444ef8afccd8b23e0799b52", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/archive/f83cb8dd6f5658bc574ccd873e25197055265d1c.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/archive/f83cb8dd6f5658bc574ccd873e25197055265d1c.tar.gz" + ] + }, + "io_bazel_rules_sass": { + "name": "io_bazel_rules_sass", + "sha256": "d868ce50d592ef4aad7dec4dd32ae68d2151261913450fac8390b3fd474bb898", + "strip_prefix": "rules_sass-8ccf4f1c351928b55d5dddf3672e3667f6978d60", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_sass/archive/8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz", + "https://github.com/bazelbuild/rules_sass/archive/8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz" + ] + }, + "io_bazel_skydoc": { + "name": "io_bazel_skydoc", + "sha256": "fdc34621839104b57363a258eab9d821b02ff7837923cfe7fb6fd67182780829", + "strip_prefix": "skydoc-41c28e43dffbae39c52dd4b91932d1209e5a8893", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/skydoc/archive/41c28e43dffbae39c52dd4b91932d1209e5a8893.tar.gz", + "https://github.com/bazelbuild/skydoc/archive/41c28e43dffbae39c52dd4b91932d1209e5a8893.tar.gz" + ] + }, + "java_tools_javac11_darwin-v4.0.zip": { + "name": "java_tools_javac11_darwin-v4.0.zip", + "sha256": "fbf5bf22e9aab9c622e4c8c59314a1eef5ea09eafc5672b4f3250dc0b971bbcc", + "urls": [ + "https://mirror.bazel.build/bazel_java_tools/releases/javac11/v4.0/java_tools_javac11_darwin-v4.0.zip" + ] + }, + "java_tools_javac11_linux-v4.0.zip": { + "name": "java_tools_javac11_linux-v4.0.zip", + "sha256": "96e223094a12c842a66db0bb7bb6866e88e26e678f045842911f9bd6b47161f5", + "urls": [ + "https://mirror.bazel.build/bazel_java_tools/releases/javac11/v4.0/java_tools_javac11_linux-v4.0.zip" + ] + }, + "java_tools_javac11_windows-v4.0.zip": { + "name": "java_tools_javac11_windows-v4.0.zip", + "sha256": "a1de51447b2ba2eab923d589ba6c72c289c16e6091e6a3bb3e67a05ef4ad200c", + "urls": [ + "https://mirror.bazel.build/bazel_java_tools/releases/javac11/v4.0/java_tools_javac11_windows-v4.0.zip" + ] + }, + "java_tools_langtools_javac10": { + "name": "java_tools_langtools_javac10", + "sha256": "0e9c9ac5ef17869de3cb8c3497c4c0d31836ef7b63efe1690506f53783adb212", + "urls": [ + "https://mirror.bazel.build/bazel_java_tools/jdk_langtools/langtools_jdk10_v2.zip" + ] + }, + "java_tools_langtools_javac11": { + "name": "java_tools_langtools_javac11", + "sha256": "cf0814fa002ef3d794582bb086516d8c9ed0958f83f19799cdb08949019fe4c7", + "urls": [ + "https://mirror.bazel.build/bazel_java_tools/jdk_langtools/langtools_jdk11_v2.zip" + ] + }, + "java_tools_langtools_javac12": { + "name": "java_tools_langtools_javac12", + "sha256": "99b107105165a91df82cd7cf82a8efb930d803fb7de1663cf7f780142104cd14", + "urls": [ + "https://mirror.bazel.build/bazel_java_tools/jdk_langtools/langtools_jdk12.zip" + ] + }, + "java_tools_langtools_javac9": { + "name": "java_tools_langtools_javac9", + "sha256": "d94befcfb325a9a62aebc2052e631fde2322b4df5c82a19ed260b38ba12a0ad1", + "urls": [ + "https://mirror.bazel.build/bazel_java_tools/jdk_langtools/langtools_jdk9_v2.zip" + ] + }, + "jdk10-server-release-1804.tar.xz": { + "name": "jdk10-server-release-1804.tar.xz", + "sha256": "b7098b7aaf6ee1ffd4a2d0371a0be26c5a5c87f6aebbe46fe9a92c90583a84be", + "urls": [ + "https://mirror.bazel.build/openjdk.linaro.org/releases/jdk10-server-release-1804.tar.xz" + ] + }, + "jdk9-server-release-1708.tar.xz": { + "name": "jdk9-server-release-1708.tar.xz", + "sha256": "72e7843902b0395e2d30e1e9ad2a5f05f36a4bc62529828bcbc698d54aec6022", + "urls": [ + "https://mirror.bazel.build/openjdk.linaro.org/releases/jdk9-server-release-1708.tar.xz" + ] + }, + "openjdk10_linux_archive": { + "build_file_content": "java_runtime(name = 'runtime', srcs = glob(['**']), visibility = ['//visibility:public'])", + "name": "openjdk10_linux_archive", + "sha256": "b3c2d762091a615b0c1424ebbd05d75cc114da3bf4f25a0dec5c51ea7e84146f", + "strip_prefix": "zulu10.2+3-jdk10.0.1-linux_x64", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu10.2+3-jdk10.0.1/zulu10.2+3-jdk10.0.1-linux_x64.tar.gz" + ] + }, + "openjdk11_linux_archive": { + "build_file_content": "java_runtime(name = 'runtime', srcs = glob(['**']), visibility = ['//visibility:public'])", + "name": "openjdk11_linux_archive", + "sha256": "ddb0fd4526089cf1ce2db36282c282263f587a9e8be373fa02f511a12923cc48", + "strip_prefix": "zulu11.31.11-ca-jdk11.0.3-linux_x64", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.31.11-ca-jdk11.0.3/zulu11.31.11-ca-jdk11.0.3-linux_x64.tar.gz" + ] + }, + "openjdk12_linux_archive": { + "build_file_content": "java_runtime(name = 'runtime', srcs = glob(['**']), visibility = ['//visibility:public'])", + "name": "openjdk12_linux_archive", + "sha256": "529c99841d69e11a85aea967ccfb9d0fd40b98c5b68dbe1d059002655e0a9c13", + "strip_prefix": "zulu12.2.3-ca-jdk12.0.1-linux_x64", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu12.2.3-ca-jdk12.0.1/zulu12.2.3-ca-jdk12.0.1-linux_x64.tar.gz" + ] + }, + "openjdk9_linux_archive": { + "build_file_content": "java_runtime(name = 'runtime', srcs = glob(['**']), visibility = ['//visibility:public'])", + "name": "openjdk9_linux_archive", + "sha256": "45f2dfbee93b91b1468cf81d843fc6d9a47fef1f831c0b7ceff4f1eb6e6851c8", + "strip_prefix": "zulu9.0.7.1-jdk9.0.7-linux_x64", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-linux_x64.tar.gz" + ] + }, + "openjdk_linux": { + "downloaded_file_path": "zulu-linux.tar.gz", + "name": "openjdk_linux", + "sha256": "460d8a4f0c0204160b48086e341b22943c9cca471b195340e75b38ae9eb33c1c", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-linux_x64-allmodules-90755145cb6e6418584d8603cd5fa9afbb30aecc-1549209950.tar.gz" + ] + }, + "openjdk_linux_aarch64": { + "downloaded_file_path": "zulu-linux-aarch64.tar.gz", + "name": "openjdk_linux_aarch64", + "sha256": "23c37c0c3a8fdcbc68e96e70ff5c5c020c14db76deaae9b547849afda4586e5e", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.31.15-ca-jdk11.0.3/zulu11.31.15-ca-jdk11.0.3-linux_aarch64-allmodules-c82eb4878c7dc829455caeb915affe36c89df06f-1561630858.tar.gz" + ] + }, + "openjdk_linux_aarch64_minimal": { + "downloaded_file_path": "zulu-linux-aarch64-minimal.tar.gz", + "name": "openjdk_linux_aarch64_minimal", + "sha256": "7af2583fe5ef0a781d4a9dca0c0160d42e7db1305ec1b66f98aa44c91cc875df", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.31.15-ca-jdk11.0.3/zulu11.31.15-ca-jdk11.0.3-linux_aarch64-minimal-c82eb4878c7dc829455caeb915affe36c89df06f-1561630858.tar.gz" + ] + }, + "openjdk_linux_aarch64_vanilla": { + "downloaded_file_path": "zulu-linux-aarch64-vanilla.tar.gz", + "name": "openjdk_linux_aarch64_vanilla", + "sha256": "3b0d91611b1bdc4d409afcf9eab4f0e7f4ae09f88fc01bd9f2b48954882ae69b", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.31.15-ca-jdk11.0.3/zulu11.31.15-ca-jdk11.0.3-linux_aarch64.tar.gz" + ] + }, + "openjdk_linux_minimal": { + "downloaded_file_path": "zulu-linux-minimal.tar.gz", + "name": "openjdk_linux_minimal", + "sha256": "5123bc8dd21886761d1fd9ca0fb1898b3372d7243064a070ec81ca9c9d1a6791", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-linux_x64-minimal-524ae2ca2a782c9f15e00f08bd35b3f8ceacbd7f-1556011926.tar.gz" + ] + }, + "openjdk_linux_vanilla": { + "downloaded_file_path": "zulu-linux-vanilla.tar.gz", + "name": "openjdk_linux_vanilla", + "sha256": "f3f44b6235508e87b760bf37a49e186cc1fa4e9cd28384c4dbf5a33991921e08", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-linux_x64.tar.gz" + ] + }, + "openjdk_macos": { + "downloaded_file_path": "zulu-macos.tar.gz", + "name": "openjdk_macos", + "sha256": "8fa61d85ca6f657d646fdb50cfc8634987f8f7d8a3250ed39fb7364647633252", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-macosx_x64-allmodules-90755145cb6e6418584d8603cd5fa9afbb30aecc-1549209951.tar.gz" + ] + }, + "openjdk_macos_minimal": { + "downloaded_file_path": "zulu-macos-minimal.tar.gz", + "name": "openjdk_macos_minimal", + "sha256": "ac56e44db46fd56ac78b39b6823daed4faa74a2677ac340c7d217f863884ec0f", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-macosx_x64-minimal-524ae2ca2a782c9f15e00f08bd35b3f8ceacbd7f-1556003114.tar.gz" + ] + }, + "openjdk_macos_vanilla": { + "downloaded_file_path": "zulu-macos-vanilla.tar.gz", + "name": "openjdk_macos_vanilla", + "sha256": "059f8e3484bf07b63a8f2820d5f528f473eff1befdb1896ee4f8ff06be3b8d8f", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-macosx_x64.zip" + ] + }, + "openjdk_win": { + "downloaded_file_path": "zulu-win.zip", + "name": "openjdk_win", + "sha256": "e6ddb361309f8e84eb5fb5ad8b0f5cc031ba3679910139262c31efd8f7579d05", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-win_x64-allmodules-90755145cb6e6418584d8603cd5fa9afbb30aecc-1549209972.zip" + ] + }, + "openjdk_win_minimal": { + "downloaded_file_path": "zulu-win-minimal.zip", + "name": "openjdk_win_minimal", + "sha256": "8e5dada6e9ebcc9ce29b4d051449bb95d3ee1e620e166da862224bbf15211f8b", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-win_x64-minimal-524ae2ca2a782c9f15e00f08bd35b3f8ceacbd7f-1556003136.zip" + ] + }, + "openjdk_win_vanilla": { + "downloaded_file_path": "zulu-win-vanilla.zip", + "name": "openjdk_win_vanilla", + "sha256": "e1f5b4ce1b9148140fae2fcfb8a96d1c9b7eac5b8df0e13fbcad9b8561284880", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-win_x64.zip" + ] + }, + "platforms": { + "name": "platforms", + "sha256": "a07fe5e75964361885db725039c2ba673f0ee0313d971ae4f50c9b18cd28b0b5", + "strip_prefix": "platforms-441afe1bfdadd6236988e9cac159df6b5a9f5a98", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip", + "https://github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip" + ] + }, + "rules_cc": { + "name": "rules_cc", + "sha256": "36fa66d4d49debd71d05fba55c1353b522e8caef4a20f8080a3d17cdda001d89", + "strip_prefix": "rules_cc-0d5f3f2768c6ca2faca0079a997a97ce22997a0c", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_cc/archive/0d5f3f2768c6ca2faca0079a997a97ce22997a0c.zip", + "https://github.com/bazelbuild/rules_cc/archive/0d5f3f2768c6ca2faca0079a997a97ce22997a0c.zip" + ] + }, + "rules_java": { + "name": "rules_java", + "sha256": "bc81f1ba47ef5cc68ad32225c3d0e70b8c6f6077663835438da8d5733f917598", + "strip_prefix": "rules_java-7cf3cefd652008d0a64a419c34c13bdca6c8f178", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_java/archive/7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip", + "https://github.com/bazelbuild/rules_java/archive/7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip" + ] + }, + "rules_pkg": { + "name": "rules_pkg", + "sha256": "5bdc04987af79bd27bc5b00fe30f59a858f77ffa0bd2d8143d5b31ad8b1bd71c", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/rules_pkg-0.2.0.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.0/rules_pkg-0.2.0.tar.gz" + ] + }, + "rules_pkg-0.2.0.tar.gz": { + "name": "rules_pkg-0.2.0.tar.gz", + "sha256": "5bdc04987af79bd27bc5b00fe30f59a858f77ffa0bd2d8143d5b31ad8b1bd71c", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/rules_pkg-0.2.0.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.0/rules_pkg-0.2.0.tar.gz" + ] + }, + "rules_proto": { + "name": "rules_proto", + "sha256": "88b0a90433866b44bb4450d4c30bc5738b8c4f9c9ba14e9661deb123f56a833d", + "strip_prefix": "rules_proto-b0cc14be5da05168b01db282fe93bdf17aa2b9f4", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/b0cc14be5da05168b01db282fe93bdf17aa2b9f4.tar.gz", + "https://github.com/bazelbuild/rules_proto/archive/b0cc14be5da05168b01db282fe93bdf17aa2b9f4.tar.gz" + ] + }, + "zulu10.2+3-jdk10.0.1-linux_x64-allmodules.tar.gz": { + "name": "zulu10.2+3-jdk10.0.1-linux_x64-allmodules.tar.gz", + "sha256": "57fad3602e74c79587901d6966d3b54ef32cb811829a2552163185d5064fe9b5", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu10.2+3-jdk10.0.1/zulu10.2+3-jdk10.0.1-linux_x64-allmodules.tar.gz" + ] + }, + "zulu10.2+3-jdk10.0.1-macosx_x64-allmodules.tar.gz": { + "name": "zulu10.2+3-jdk10.0.1-macosx_x64-allmodules.tar.gz", + "sha256": "e669c9a897413d855b550b4e39d79614392e6fb96f494e8ef99a34297d9d85d3", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu10.2+3-jdk10.0.1/zulu10.2+3-jdk10.0.1-macosx_x64-allmodules.tar.gz" + ] + }, + "zulu10.2+3-jdk10.0.1-win_x64-allmodules.zip": { + "name": "zulu10.2+3-jdk10.0.1-win_x64-allmodules.zip", + "sha256": "c39e7700a8d41794d60985df5a20352435196e78ecbc6a2b30df7be8637bffd5", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu10.2+3-jdk10.0.1/zulu10.2+3-jdk10.0.1-win_x64-allmodules.zip" + ] + }, + "zulu11.2.3-jdk11.0.1-linux_x64.tar.gz": { + "name": "zulu11.2.3-jdk11.0.1-linux_x64.tar.gz", + "sha256": "232b1c3511f0d26e92582b7c3cc363be7ac633e371854ca2f2e9f2b50eb72a75", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.2.3-jdk11.0.1/zulu11.2.3-jdk11.0.1-linux_x64.tar.gz" + ] + }, + "zulu11.2.3-jdk11.0.1-macosx_x64.tar.gz": { + "name": "zulu11.2.3-jdk11.0.1-macosx_x64.tar.gz", + "sha256": "1edf366ee821e5db8e348152fcb337b28dfd6bf0f97943c270dcc6747cedb6cb", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.2.3-jdk11.0.1/zulu11.2.3-jdk11.0.1-macosx_x64.tar.gz" + ] + }, + "zulu11.2.3-jdk11.0.1-win_x64.zip": { + "name": "zulu11.2.3-jdk11.0.1-win_x64.zip", + "sha256": "8e1e2b8347de6746f3fd1538840dd643201533ab113abc4ed93678e342d28aa3", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.2.3-jdk11.0.1/zulu11.2.3-jdk11.0.1-win_x64.zip" + ] + }, + "zulu11.29.3-ca-jdk11.0.2-linux_x64.tar.gz": { + "name": "zulu11.29.3-ca-jdk11.0.2-linux_x64.tar.gz", + "sha256": "f3f44b6235508e87b760bf37a49e186cc1fa4e9cd28384c4dbf5a33991921e08", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-linux_x64.tar.gz" + ] + }, + "zulu11.29.3-ca-jdk11.0.2-macosx_x64.zip": { + "name": "zulu11.29.3-ca-jdk11.0.2-macosx_x64.zip", + "sha256": "059f8e3484bf07b63a8f2820d5f528f473eff1befdb1896ee4f8ff06be3b8d8f", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-macosx_x64.zip" + ] + }, + "zulu11.29.3-ca-jdk11.0.2-win_x64.zip": { + "name": "zulu11.29.3-ca-jdk11.0.2-win_x64.zip", + "sha256": "e1f5b4ce1b9148140fae2fcfb8a96d1c9b7eac5b8df0e13fbcad9b8561284880", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-win_x64.zip" + ] + }, + "zulu11.31.15-ca-jdk11.0.3-linux_aarch64.tar.gz": { + "name": "zulu11.31.15-ca-jdk11.0.3-linux_aarch64.tar.gz", + "sha256": "3b0d91611b1bdc4d409afcf9eab4f0e7f4ae09f88fc01bd9f2b48954882ae69b", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu11.31.15-ca-jdk11.0.3/zulu11.31.15-ca-jdk11.0.3-linux_aarch64.tar.gz" + ] + }, + "zulu9.0.7.1-jdk9.0.7-linux_x64-allmodules.tar.gz": { + "name": "zulu9.0.7.1-jdk9.0.7-linux_x64-allmodules.tar.gz", + "sha256": "f27cb933de4f9e7fe9a703486cf44c84bc8e9f138be0c270c9e5716a32367e87", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-linux_x64-allmodules.tar.gz" + ] + }, + "zulu9.0.7.1-jdk9.0.7-macosx_x64-allmodules.tar.gz": { + "name": "zulu9.0.7.1-jdk9.0.7-macosx_x64-allmodules.tar.gz", + "sha256": "404e7058ff91f956612f47705efbee8e175a38b505fb1b52d8c1ea98718683de", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-macosx_x64-allmodules.tar.gz" + ] + }, + "zulu9.0.7.1-jdk9.0.7-win_x64-allmodules.zip": { + "name": "zulu9.0.7.1-jdk9.0.7-win_x64-allmodules.zip", + "sha256": "e738829017f107e7a7cd5069db979398ec3c3f03ef56122f89ba38e7374f63ed", + "urls": [ + "https://mirror.bazel.build/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-win_x64-allmodules.zip" + ] + } +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 7ee2d3fb0f51..9d5c7fbb41d3 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -9707,7 +9707,18 @@ in stdenv = if stdenv.cc.isClang then llvmPackages_6.stdenv else stdenv; }; - bazel_0 = callPackage ../development/tools/build-managers/bazel/bazel_0 { + bazel_0 = bazel_0_26; + + bazel_0_26 = callPackage ../development/tools/build-managers/bazel/bazel_0_26 { + inherit (darwin) cctools; + inherit (darwin.apple_sdk.frameworks) CoreFoundation CoreServices Foundation; + buildJdk = jdk8; + buildJdkName = "jdk8"; + runJdk = jdk11_headless; + stdenv = if stdenv.cc.isClang then llvmPackages_6.stdenv else stdenv; + }; + + bazel_0_29 = callPackage ../development/tools/build-managers/bazel/bazel_0_29 { inherit (darwin) cctools; inherit (darwin.apple_sdk.frameworks) CoreFoundation CoreServices Foundation; buildJdk = jdk8; -- cgit 1.4.1 From a730888ce0c608789fbc57da7449c6a618b3e906 Mon Sep 17 00:00:00 2001 From: Timo Kaufmann Date: Tue, 19 Nov 2019 22:45:06 +0100 Subject: python.pkgs.tensorflow{,-bin,-estimator,-tensorboard}-2: Init at 2.1.0 Major breaking change from 1.x, so treating keeping both versions for now. (adapted from 33f11be707e39acf96423f97f3baa80d8f11a0cb) (adapted from 9e8dea7986dbdde850a58c7704182776642d8919) --- .../tensorflow-estimator/1/default.nix | 27 ++ .../python-modules/tensorflow-estimator/1_15_1.nix | 27 -- .../tensorflow-estimator/2/default.nix | 27 ++ .../tensorflow-estimator/default.nix | 27 -- .../tensorflow-tensorboard/1/default.nix | 66 ++++ .../tensorflow-tensorboard/2/default.nix | 68 ++++ .../tensorflow-tensorboard/default.nix | 66 ---- .../python-modules/tensorflow/1/bin.nix | 121 ++++++ .../python-modules/tensorflow/1/binary-hashes.nix | 43 ++ .../python-modules/tensorflow/1/default.nix | 430 ++++++++++++++++++++ .../tensorflow/1/lift-gast-restriction.patch | 13 + .../python-modules/tensorflow/1/prefetcher.sh | 33 ++ .../python-modules/tensorflow/2/bin.nix | 179 +++++++++ .../python-modules/tensorflow/2/binary-hashes.nix | 51 +++ .../python-modules/tensorflow/2/default.nix | 421 ++++++++++++++++++++ .../tensorflow/2/lift-gast-restriction.patch | 11 + .../python-modules/tensorflow/2/prefetcher.sh | 44 +++ pkgs/development/python-modules/tensorflow/bin.nix | 121 ------ .../python-modules/tensorflow/binary-hashes.nix | 43 -- .../python-modules/tensorflow/default.nix | 431 --------------------- .../tensorflow/lift-gast-restriction.patch | 13 - .../python-modules/tensorflow/prefetcher.sh | 33 -- pkgs/top-level/python-packages.nix | 42 +- 23 files changed, 1570 insertions(+), 767 deletions(-) create mode 100644 pkgs/development/python-modules/tensorflow-estimator/1/default.nix delete mode 100644 pkgs/development/python-modules/tensorflow-estimator/1_15_1.nix create mode 100644 pkgs/development/python-modules/tensorflow-estimator/2/default.nix delete mode 100644 pkgs/development/python-modules/tensorflow-estimator/default.nix create mode 100644 pkgs/development/python-modules/tensorflow-tensorboard/1/default.nix create mode 100644 pkgs/development/python-modules/tensorflow-tensorboard/2/default.nix delete mode 100644 pkgs/development/python-modules/tensorflow-tensorboard/default.nix create mode 100644 pkgs/development/python-modules/tensorflow/1/bin.nix create mode 100644 pkgs/development/python-modules/tensorflow/1/binary-hashes.nix create mode 100644 pkgs/development/python-modules/tensorflow/1/default.nix create mode 100644 pkgs/development/python-modules/tensorflow/1/lift-gast-restriction.patch create mode 100755 pkgs/development/python-modules/tensorflow/1/prefetcher.sh create mode 100644 pkgs/development/python-modules/tensorflow/2/bin.nix create mode 100644 pkgs/development/python-modules/tensorflow/2/binary-hashes.nix create mode 100644 pkgs/development/python-modules/tensorflow/2/default.nix create mode 100644 pkgs/development/python-modules/tensorflow/2/lift-gast-restriction.patch create mode 100755 pkgs/development/python-modules/tensorflow/2/prefetcher.sh delete mode 100644 pkgs/development/python-modules/tensorflow/bin.nix delete mode 100644 pkgs/development/python-modules/tensorflow/binary-hashes.nix delete mode 100644 pkgs/development/python-modules/tensorflow/default.nix delete mode 100644 pkgs/development/python-modules/tensorflow/lift-gast-restriction.patch delete mode 100755 pkgs/development/python-modules/tensorflow/prefetcher.sh diff --git a/pkgs/development/python-modules/tensorflow-estimator/1/default.nix b/pkgs/development/python-modules/tensorflow-estimator/1/default.nix new file mode 100644 index 000000000000..39c667357c8f --- /dev/null +++ b/pkgs/development/python-modules/tensorflow-estimator/1/default.nix @@ -0,0 +1,27 @@ +{ stdenv, fetchPypi, buildPythonPackage +, numpy +, absl-py +, mock +}: + +buildPythonPackage rec { + pname = "tensorflow-estimator"; + version = "1.15.1"; + format = "wheel"; + + src = fetchPypi { + pname = "tensorflow_estimator"; + inherit version format; + sha256 = "1fc61wmc0w22frs79j2x4g6wnv5g21xc6rix1g4bsvy9qfvvylw8"; + }; + + propagatedBuildInputs = [ mock numpy absl-py ]; + + meta = with stdenv.lib; { + description = "TensorFlow Estimator is a high-level API that encapsulates model training, evaluation, prediction, and exporting."; + homepage = http://tensorflow.org; + license = licenses.asl20; + maintainers = with maintainers; [ jyp ]; + }; +} + diff --git a/pkgs/development/python-modules/tensorflow-estimator/1_15_1.nix b/pkgs/development/python-modules/tensorflow-estimator/1_15_1.nix deleted file mode 100644 index 39c667357c8f..000000000000 --- a/pkgs/development/python-modules/tensorflow-estimator/1_15_1.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ stdenv, fetchPypi, buildPythonPackage -, numpy -, absl-py -, mock -}: - -buildPythonPackage rec { - pname = "tensorflow-estimator"; - version = "1.15.1"; - format = "wheel"; - - src = fetchPypi { - pname = "tensorflow_estimator"; - inherit version format; - sha256 = "1fc61wmc0w22frs79j2x4g6wnv5g21xc6rix1g4bsvy9qfvvylw8"; - }; - - propagatedBuildInputs = [ mock numpy absl-py ]; - - meta = with stdenv.lib; { - description = "TensorFlow Estimator is a high-level API that encapsulates model training, evaluation, prediction, and exporting."; - homepage = http://tensorflow.org; - license = licenses.asl20; - maintainers = with maintainers; [ jyp ]; - }; -} - diff --git a/pkgs/development/python-modules/tensorflow-estimator/2/default.nix b/pkgs/development/python-modules/tensorflow-estimator/2/default.nix new file mode 100644 index 000000000000..47ce0f83b02d --- /dev/null +++ b/pkgs/development/python-modules/tensorflow-estimator/2/default.nix @@ -0,0 +1,27 @@ +{ stdenv, fetchPypi, buildPythonPackage +, numpy +, absl-py +, mock +}: + +buildPythonPackage rec { + pname = "tensorflow-estimator"; + version = "2.1.0"; + format = "wheel"; + + src = fetchPypi { + pname = "tensorflow_estimator"; + inherit version format; + sha256 = "0wk9viil54ms1s2ir7zxygqa425i69hx8zngwhdqvw9nlr4gdig5"; + }; + + propagatedBuildInputs = [ mock numpy absl-py ]; + + meta = with stdenv.lib; { + description = "TensorFlow Estimator is a high-level API that encapsulates model training, evaluation, prediction, and exporting."; + homepage = http://tensorflow.org; + license = licenses.asl20; + maintainers = with maintainers; [ jyp ]; + }; +} + diff --git a/pkgs/development/python-modules/tensorflow-estimator/default.nix b/pkgs/development/python-modules/tensorflow-estimator/default.nix deleted file mode 100644 index 47ce0f83b02d..000000000000 --- a/pkgs/development/python-modules/tensorflow-estimator/default.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ stdenv, fetchPypi, buildPythonPackage -, numpy -, absl-py -, mock -}: - -buildPythonPackage rec { - pname = "tensorflow-estimator"; - version = "2.1.0"; - format = "wheel"; - - src = fetchPypi { - pname = "tensorflow_estimator"; - inherit version format; - sha256 = "0wk9viil54ms1s2ir7zxygqa425i69hx8zngwhdqvw9nlr4gdig5"; - }; - - propagatedBuildInputs = [ mock numpy absl-py ]; - - meta = with stdenv.lib; { - description = "TensorFlow Estimator is a high-level API that encapsulates model training, evaluation, prediction, and exporting."; - homepage = http://tensorflow.org; - license = licenses.asl20; - maintainers = with maintainers; [ jyp ]; - }; -} - diff --git a/pkgs/development/python-modules/tensorflow-tensorboard/1/default.nix b/pkgs/development/python-modules/tensorflow-tensorboard/1/default.nix new file mode 100644 index 000000000000..e0108aed5ef6 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow-tensorboard/1/default.nix @@ -0,0 +1,66 @@ +{ lib, fetchPypi, buildPythonPackage, isPy3k +, numpy +, wheel +, werkzeug +, protobuf +, grpcio +, markdown +, futures +, absl-py +}: + +# tensorflow/tensorboard is built from a downloaded wheel, because +# https://github.com/tensorflow/tensorboard/issues/719 blocks +# buildBazelPackage. + +buildPythonPackage rec { + pname = "tensorflow-tensorboard"; + version = "1.15.0"; + format = "wheel"; + + src = fetchPypi ({ + pname = "tensorboard"; + inherit version; + format = "wheel"; + } // (if isPy3k then { + python = "py3"; + sha256 = "1g62i3nrgp8q9wfsyqqjkkfnsz7x2k018c26kdh527h1yrjjrbac"; + } else { + python = "py2"; + sha256 = "0l3zc8j2sh7h1z4qpy8kfvclv3kzndri55p10i42q6xahs9phav1"; + })); + + propagatedBuildInputs = [ + numpy + werkzeug + protobuf + markdown + grpcio + absl-py + # not declared in install_requires, but used at runtime + # https://github.com/NixOS/nixpkgs/issues/73840 + wheel + ] ++ lib.optional (!isPy3k) futures; + + # in the absence of a real test suite, run cli and imports + checkPhase = '' + $out/bin/tensorboard --help > /dev/null + ''; + + pythonImportsCheck = [ + "tensorboard" + "tensorboard.backend" + "tensorboard.compat" + "tensorboard.data" + "tensorboard.plugins" + "tensorboard.summary" + "tensorboard.util" + ]; + + meta = with lib; { + description = "TensorFlow's Visualization Toolkit"; + homepage = http://tensorflow.org; + license = licenses.asl20; + maintainers = with maintainers; [ abbradar ]; + }; +} diff --git a/pkgs/development/python-modules/tensorflow-tensorboard/2/default.nix b/pkgs/development/python-modules/tensorflow-tensorboard/2/default.nix new file mode 100644 index 000000000000..202b167c7e6c --- /dev/null +++ b/pkgs/development/python-modules/tensorflow-tensorboard/2/default.nix @@ -0,0 +1,68 @@ +{ lib, fetchPypi, buildPythonPackage, isPy3k +, numpy +, wheel +, werkzeug +, protobuf +, grpcio +, markdown +, futures +, absl-py +, google-auth-oauthlib +}: + +# tensorflow/tensorboard is built from a downloaded wheel, because +# https://github.com/tensorflow/tensorboard/issues/719 blocks +# buildBazelPackage. + +buildPythonPackage rec { + pname = "tensorflow-tensorboard"; + version = "2.1.0"; + format = "wheel"; + + src = fetchPypi ({ + pname = "tensorboard"; + inherit version; + format = "wheel"; + } // (if isPy3k then { + python = "py3"; + sha256 = "1wpjdzhjpcdkyaahzd4bl71k4l30z5c55280ndiwj32hw70lxrp6"; + } else { + python = "py2"; + sha256 = "1f805839xa36wxb7xac9fyxzaww92vw4d50vs6g61wnlr4byp00w"; + })); + + propagatedBuildInputs = [ + numpy + werkzeug + protobuf + markdown + grpcio + absl-py + google-auth-oauthlib + # not declared in install_requires, but used at runtime + # https://github.com/NixOS/nixpkgs/issues/73840 + wheel + ] ++ lib.optional (!isPy3k) futures; + + # in the absence of a real test suite, run cli and imports + checkPhase = '' + $out/bin/tensorboard --help > /dev/null + ''; + + pythonImportsCheck = [ + "tensorboard" + "tensorboard.backend" + "tensorboard.compat" + "tensorboard.data" + "tensorboard.plugins" + "tensorboard.summary" + "tensorboard.util" + ]; + + meta = with lib; { + description = "TensorFlow's Visualization Toolkit"; + homepage = http://tensorflow.org; + license = licenses.asl20; + maintainers = with maintainers; [ abbradar ]; + }; +} diff --git a/pkgs/development/python-modules/tensorflow-tensorboard/default.nix b/pkgs/development/python-modules/tensorflow-tensorboard/default.nix deleted file mode 100644 index e0108aed5ef6..000000000000 --- a/pkgs/development/python-modules/tensorflow-tensorboard/default.nix +++ /dev/null @@ -1,66 +0,0 @@ -{ lib, fetchPypi, buildPythonPackage, isPy3k -, numpy -, wheel -, werkzeug -, protobuf -, grpcio -, markdown -, futures -, absl-py -}: - -# tensorflow/tensorboard is built from a downloaded wheel, because -# https://github.com/tensorflow/tensorboard/issues/719 blocks -# buildBazelPackage. - -buildPythonPackage rec { - pname = "tensorflow-tensorboard"; - version = "1.15.0"; - format = "wheel"; - - src = fetchPypi ({ - pname = "tensorboard"; - inherit version; - format = "wheel"; - } // (if isPy3k then { - python = "py3"; - sha256 = "1g62i3nrgp8q9wfsyqqjkkfnsz7x2k018c26kdh527h1yrjjrbac"; - } else { - python = "py2"; - sha256 = "0l3zc8j2sh7h1z4qpy8kfvclv3kzndri55p10i42q6xahs9phav1"; - })); - - propagatedBuildInputs = [ - numpy - werkzeug - protobuf - markdown - grpcio - absl-py - # not declared in install_requires, but used at runtime - # https://github.com/NixOS/nixpkgs/issues/73840 - wheel - ] ++ lib.optional (!isPy3k) futures; - - # in the absence of a real test suite, run cli and imports - checkPhase = '' - $out/bin/tensorboard --help > /dev/null - ''; - - pythonImportsCheck = [ - "tensorboard" - "tensorboard.backend" - "tensorboard.compat" - "tensorboard.data" - "tensorboard.plugins" - "tensorboard.summary" - "tensorboard.util" - ]; - - meta = with lib; { - description = "TensorFlow's Visualization Toolkit"; - homepage = http://tensorflow.org; - license = licenses.asl20; - maintainers = with maintainers; [ abbradar ]; - }; -} diff --git a/pkgs/development/python-modules/tensorflow/1/bin.nix b/pkgs/development/python-modules/tensorflow/1/bin.nix new file mode 100644 index 000000000000..05d59f9ed0ae --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/1/bin.nix @@ -0,0 +1,121 @@ +{ stdenv +, lib +, fetchurl +, buildPythonPackage +, isPy3k, pythonOlder, isPy38 +, astor +, gast +, google-pasta +, wrapt +, numpy +, six +, termcolor +, protobuf +, absl-py +, grpcio +, mock +, backports_weakref +, tensorflow-estimator_1 +, tensorflow-tensorboard +, cudaSupport ? false +, cudatoolkit ? null +, cudnn ? null +, nvidia_x11 ? null +, zlib +, python +, symlinkJoin +, keras-applications +, keras-preprocessing +, addOpenGLRunpath +}: + +# We keep this binary build for two reasons: +# - the source build doesn't work on Darwin. +# - the source build is currently brittle and not easy to maintain + +assert cudaSupport -> cudatoolkit != null + && cudnn != null + && nvidia_x11 != null; + +# unsupported combination +assert ! (stdenv.isDarwin && cudaSupport); + +let + packages = import ./binary-hashes.nix; + + variant = if cudaSupport then "-gpu" else ""; + pname = "tensorflow${variant}"; + +in buildPythonPackage { + inherit pname; + inherit (packages) version; + format = "wheel"; + + disabled = isPy38; + + src = let + pyVerNoDot = lib.strings.stringAsChars (x: if x == "." then "" else x) python.pythonVersion; + pyver = if stdenv.isDarwin then builtins.substring 0 1 pyVerNoDot else pyVerNoDot; + platform = if stdenv.isDarwin then "mac" else "linux"; + unit = if cudaSupport then "gpu" else "cpu"; + key = "${platform}_py_${pyver}_${unit}"; + in fetchurl packages.${key}; + + propagatedBuildInputs = [ + protobuf + numpy + termcolor + grpcio + six + astor + absl-py + gast + google-pasta + wrapt + tensorflow-estimator_1 + tensorflow-tensorboard + keras-applications + keras-preprocessing + ] ++ lib.optional (!isPy3k) mock + ++ lib.optionals (pythonOlder "3.4") [ backports_weakref ]; + + nativeBuildInputs = lib.optional cudaSupport addOpenGLRunpath; + + # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow + # and the propageted input tensorflow-tensorboard which causes environment collisions. + # another possibility would be to have tensorboard only in the buildInputs + # https://github.com/tensorflow/tensorflow/blob/v1.7.1/tensorflow/tools/pip_package/setup.py#L79 + postInstall = '' + rm $out/bin/tensorboard + ''; + + # Note that we need to run *after* the fixup phase because the + # libraries are loaded at runtime. If we run in preFixup then + # patchelf --shrink-rpath will remove the cuda libraries. + postFixup = let + rpath = stdenv.lib.makeLibraryPath + ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]); + in + lib.optionalString stdenv.isLinux '' + rrPath="$out/${python.sitePackages}/tensorflow/:$out/${python.sitePackages}/tensorflow/contrib/tensor_forest/:${rpath}" + internalLibPath="$out/${python.sitePackages}/tensorflow/python/_pywrap_tensorflow_internal.so" + find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do + patchelf --set-rpath "$rrPath" "$lib" + ${lib.optionalString cudaSupport '' + addOpenGLRunpath "$lib" + ''} + done + ''; + + + meta = with stdenv.lib; { + description = "Computation using data flow graphs for scalable machine learning"; + homepage = http://tensorflow.org; + license = licenses.asl20; + maintainers = with maintainers; [ jyp abbradar ]; + platforms = [ "x86_64-linux" "x86_64-darwin" ]; + # Python 2.7 build uses different string encoding. + # See https://github.com/NixOS/nixpkgs/pull/37044#issuecomment-373452253 + broken = stdenv.isDarwin && !isPy3k; + }; +} diff --git a/pkgs/development/python-modules/tensorflow/1/binary-hashes.nix b/pkgs/development/python-modules/tensorflow/1/binary-hashes.nix new file mode 100644 index 000000000000..fa4809dc3f19 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/1/binary-hashes.nix @@ -0,0 +1,43 @@ +{ +version = "1.14.0"; +linux_py_27_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp27-none-linux_x86_64.whl"; + sha256 = "0yywdrfk97dh1bxhibspg0raz70fx9lcczj6xlimqy4xb60clx7k"; +}; +linux_py_35_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp35-cp35m-linux_x86_64.whl"; + sha256 = "1xvyb6xcrjhlwvrmrhn5vs9xy7g98smqmpv4i3hhpry4qyasphhj"; +}; +linux_py_36_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl"; + sha256 = "1psd9vyxz9f39dwj77nvrg373sxv3p5vdp9fnz81dpsm0b0mwl44"; +}; +linux_py_37_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp37-cp37m-linux_x86_64.whl"; + sha256 = "0bg2sb1n2ag27r7ww695kg5hb0mjrw4kc5893krmixx2j71860c5"; +}; +linux_py_27_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp27-none-linux_x86_64.whl"; + sha256 = "0y1x91gayg6pjddgl8ndcm63wfzhyv4s5khgl7ffzsgni1ivaqw5"; +}; +linux_py_35_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp35-cp35m-linux_x86_64.whl"; + sha256 = "03piggpbz1jx8m2b95spq3jrdff4w6xx63ji07am7hyw2nsgx3mx"; +}; +linux_py_36_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp36-cp36m-linux_x86_64.whl"; + sha256 = "0ypkp8cfhharsyyikb1qgf44cfm6284km9xswzvzymjzz75vg3gd"; +}; +linux_py_37_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp37-cp37m-linux_x86_64.whl"; + sha256 = "0virp8nn2ysx4855hq29kas6fm6b3dsiybwzdxy9nnb9n2d8qlm2"; +}; +mac_py_2_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py2-none-any.whl"; + sha256 = "14f86k3pgq7z6i4s4im55zpp38f0drnm7xlclavsgcc0nxnj3z26"; +}; +mac_py_3_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py3-none-any.whl"; + sha256 = "0f3swpcjfgqhj6h5wnx8snc0xjkx4hnkqx83fmlrwpncs8c131d3"; +}; +} diff --git a/pkgs/development/python-modules/tensorflow/1/default.nix b/pkgs/development/python-modules/tensorflow/1/default.nix new file mode 100644 index 000000000000..78843c612ee1 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/1/default.nix @@ -0,0 +1,430 @@ +{ stdenv, pkgs, bazel_0_26, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin +, addOpenGLRunpath +# Python deps +, buildPythonPackage, isPy3k, isPy27, pythonOlder, pythonAtLeast, python +# Python libraries +, numpy, tensorflow-tensorboard_1, backports_weakref, mock, enum34, absl-py +, future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta +, functools32 +, opt-einsum +, termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator_1 +# Common deps +, git, swig, which, binutils, glibcLocales, cython +# Common libraries +, jemalloc, openmpi, astor, gast, grpc, sqlite, openssl, jsoncpp, re2 +, curl, snappy, flatbuffers, icu, double-conversion, libpng, libjpeg, giflib +# Upsteam by default includes cuda support since tensorflow 1.15. We could do +# that in nix as well. It would make some things easier and less confusing, but +# it would also make the default tensorflow package unfree. See +# https://groups.google.com/a/tensorflow.org/forum/#!topic/developers/iRCt5m4qUz0 +, cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null, nccl ? null +# XLA without CUDA is broken +, xlaSupport ? cudaSupport +# Default from ./configure script +, cudaCapabilities ? [ "3.5" "5.2" ] +, sse42Support ? builtins.elem (stdenv.hostPlatform.platform.gcc.arch or "default") ["westmere" "sandybridge" "ivybridge" "haswell" "broadwell" "skylake" "skylake-avx512"] +, avx2Support ? builtins.elem (stdenv.hostPlatform.platform.gcc.arch or "default") [ "haswell" "broadwell" "skylake" "skylake-avx512"] +, fmaSupport ? builtins.elem (stdenv.hostPlatform.platform.gcc.arch or "default") [ "haswell" "broadwell" "skylake" "skylake-avx512"] +# Darwin deps +, Foundation, Security +}: + +assert cudaSupport -> nvidia_x11 != null + && cudatoolkit != null + && cudnn != null; + +# unsupported combination +assert ! (stdenv.isDarwin && cudaSupport); + +let + withTensorboard = pythonOlder "3.6"; + + cudatoolkit_joined = symlinkJoin { + name = "${cudatoolkit.name}-merged"; + paths = [ + cudatoolkit.lib + cudatoolkit.out + # for some reason some of the required libs are in the targets/x86_64-linux + # directory; not sure why but this works around it + "${cudatoolkit}/targets/${stdenv.system}" + ]; + }; + + cudatoolkit_cc_joined = symlinkJoin { + name = "${cudatoolkit.cc.name}-merged"; + paths = [ + cudatoolkit.cc + binutils.bintools # for ar, dwp, nm, objcopy, objdump, strip + ]; + }; + + # Needed for _some_ system libraries, grep INCLUDEDIR. + includes_joined = symlinkJoin { + name = "tensorflow-deps-merged"; + paths = [ + pkgs.protobuf + jsoncpp + ]; + }; + + tfFeature = x: if x then "1" else "0"; + + version = "1.15.2"; + variant = if cudaSupport then "-gpu" else ""; + pname = "tensorflow${variant}"; + + pythonEnv = python.withPackages (_: + [ # python deps needed during wheel build time (not runtime, see the buildPythonPackage part for that) + numpy + keras-preprocessing + protobuf + wrapt + gast + astor + absl-py + termcolor + keras-applications + setuptools + wheel + ] ++ lib.optionals (!isPy3k) + [ future + functools32 + mock + ]); + + bazel-build = buildBazelPackage { + name = "${pname}-${version}"; + bazel = bazel_0_26; + + src = fetchFromGitHub { + owner = "tensorflow"; + repo = "tensorflow"; + rev = "v${version}"; + sha256 = "1q0848drjvnaaa38dgns8knmpmkj5plzsc98j20m5ybv68s55w78"; + }; + + patches = [ + # Work around https://github.com/tensorflow/tensorflow/issues/24752 + ../no-saved-proto.patch + # Fixes for NixOS jsoncpp + ../system-jsoncpp.patch + + # https://github.com/tensorflow/tensorflow/pull/29673 + (fetchpatch { + name = "fix-compile-with-cuda-and-mpi.patch"; + url = "https://github.com/tensorflow/tensorflow/pull/29673/commits/498e35a3bfe38dd75cf1416a1a23c07c3b59e6af.patch"; + sha256 = "1m2qmwv1ysqa61z6255xggwbq6mnxbig749bdvrhnch4zydxb4di"; + }) + (fetchpatch { + name = "backport-pr-18950.patch"; + url = "https://github.com/tensorflow/tensorflow/commit/73640aaec2ab0234d9fff138e3c9833695570c0a.patch"; + sha256 = "1n9ypbrx36fc1kc9cz5b3p9qhg15xxhq4nz6ap3hwqba535nakfz"; + }) + + (fetchpatch { + # be compatible with gast >0.2 instead of only gast 0.2.2 + name = "gast-update.patch"; + url = "https://github.com/tensorflow/tensorflow/commit/85751ad6c7f5fd12c6c79545d96896cba92fa8b4.patch"; + sha256 = "077cpj0kzyqxzdya1dwh8df17zfzhqn7c685hx6iskvw2979zg2n"; + }) + ./lift-gast-restriction.patch + + # cuda 10.2 does not have "-bin2c-path" option anymore + # https://github.com/tensorflow/tensorflow/issues/34429 + ../cuda-10.2-no-bin2c-path.patch + ]; + + # On update, it can be useful to steal the changes from gentoo + # https://gitweb.gentoo.org/repo/gentoo.git/tree/sci-libs/tensorflow + + nativeBuildInputs = [ + swig which pythonEnv + ] ++ lib.optional cudaSupport addOpenGLRunpath; + + buildInputs = [ + jemalloc + openmpi + glibcLocales + git + + # libs taken from system through the TF_SYS_LIBS mechanism + # grpc + sqlite + openssl + jsoncpp + pkgs.protobuf + curl + snappy + flatbuffers + icu + double-conversion + libpng + libjpeg + giflib + re2 + pkgs.lmdb + ] ++ lib.optionals cudaSupport [ + cudatoolkit + cudnn + nvidia_x11 + ] ++ lib.optionals stdenv.isDarwin [ + Foundation + Security + ]; + + # arbitrarily set to the current latest bazel version, overly careful + TF_IGNORE_MAX_BAZEL_VERSION = true; + + # Take as many libraries from the system as possible. Keep in sync with + # list of valid syslibs in + # https://github.com/tensorflow/tensorflow/blob/master/third_party/systemlibs/syslibs_configure.bzl + TF_SYSTEM_LIBS = lib.concatStringsSep "," [ + "absl_py" + "astor_archive" + "boringssl" + # Not packaged in nixpkgs + # "com_github_googleapis_googleapis" + # "com_github_googlecloudplatform_google_cloud_cpp" + "com_google_protobuf" + "com_googlesource_code_re2" + "curl" + "cython" + "double_conversion" + "flatbuffers" + "gast_archive" + "gif_archive" + # Lots of errors, requires an older version + # "grpc" + "hwloc" + "icu" + "jpeg" + "jsoncpp_git" + "keras_applications_archive" + "lmdb" + "nasm" + # "nsync" # not packaged in nixpkgs + "opt_einsum_archive" + "org_sqlite" + "pasta" + "pcre" + "png_archive" + "six_archive" + "snappy" + "swig" + "termcolor_archive" + "wrapt" + "zlib_archive" + ]; + + INCLUDEDIR = "${includes_joined}/include"; + + PYTHON_BIN_PATH = pythonEnv.interpreter; + + TF_NEED_GCP = true; + TF_NEED_HDFS = true; + TF_ENABLE_XLA = tfFeature xlaSupport; + + CC_OPT_FLAGS = " "; + + # https://github.com/tensorflow/tensorflow/issues/14454 + TF_NEED_MPI = tfFeature cudaSupport; + + TF_NEED_CUDA = tfFeature cudaSupport; + TF_CUDA_PATHS = lib.optionalString cudaSupport "${cudatoolkit_joined},${cudnn},${nccl}"; + GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin"; + GCC_HOST_COMPILER_PATH = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin/gcc"; + TF_CUDA_COMPUTE_CAPABILITIES = lib.concatStringsSep "," cudaCapabilities; + + postPatch = '' + # https://github.com/tensorflow/tensorflow/issues/20919 + sed -i '/androidndk/d' tensorflow/lite/kernels/internal/BUILD + + # Tensorboard pulls in a bunch of dependencies, some of which may + # include security vulnerabilities. So we make it optional. + # https://github.com/tensorflow/tensorflow/issues/20280#issuecomment-400230560 + sed -i '/tensorboard >=/d' tensorflow/tools/pip_package/setup.py + ''; + + preConfigure = let + opt_flags = [] + ++ lib.optionals sse42Support ["-msse4.2"] + ++ lib.optionals avx2Support ["-mavx2"] + ++ lib.optionals fmaSupport ["-mfma"]; + in '' + patchShebangs configure + + # dummy ldconfig + mkdir dummy-ldconfig + echo "#!${stdenv.shell}" > dummy-ldconfig/ldconfig + chmod +x dummy-ldconfig/ldconfig + export PATH="$PWD/dummy-ldconfig:$PATH" + + export PYTHON_LIB_PATH="$NIX_BUILD_TOP/site-packages" + export CC_OPT_FLAGS="${lib.concatStringsSep " " opt_flags}" + mkdir -p "$PYTHON_LIB_PATH" + + # To avoid mixing Python 2 and Python 3 + unset PYTHONPATH + ''; + + configurePhase = '' + runHook preConfigure + ./configure + runHook postConfigure + ''; + + # FIXME: Tensorflow uses dlopen() for CUDA libraries. + NIX_LDFLAGS = lib.optionalString cudaSupport "-lcudart -lcublas -lcufft -lcurand -lcusolver -lcusparse -lcudnn"; + + hardeningDisable = [ "format" ]; + + bazelFlags = [ + # temporary fixes to make the build work with bazel 0.27 + "--incompatible_no_support_tools_in_action_inputs=false" + ]; + bazelBuildFlags = [ + "--config=opt" # optimize using the flags set in the configure phase + ]; + + bazelTarget = "//tensorflow/tools/pip_package:build_pip_package //tensorflow/tools/lib_package:libtensorflow"; + + fetchAttrs = { + # So that checksums don't depend on these. + TF_SYSTEM_LIBS = null; + + # cudaSupport causes fetch of ncclArchive, resulting in different hashes + sha256 = if cudaSupport then + "1qygfcvvn9vysap9nk6xccxi9mgmzyxiywz6k456f811l1v70p2c" + else + "0kfjanw0mfbh30vi1ms2xlg8yp429cbyfriik6yxd5cla2pncg2j"; + }; + + buildAttrs = { + outputs = [ "out" "python" ]; + + preBuild = '' + patchShebangs . + ''; + + installPhase = '' + mkdir -p "$out" + tar -xf bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz -C "$out" + # Write pkgconfig file. + mkdir "$out/lib/pkgconfig" + cat > "$out/lib/pkgconfig/tensorflow.pc" << EOF + Name: TensorFlow + Version: ${version} + Description: Library for computation using data flow graphs for scalable machine learning + Requires: + Libs: -L$out/lib -ltensorflow + Cflags: -I$out/include/tensorflow + EOF + + # build the source code, then copy it to $python (build_pip_package + # actually builds a symlink farm so we must dereference them). + bazel-bin/tensorflow/tools/pip_package/build_pip_package --src "$PWD/dist" + cp -Lr "$PWD/dist" "$python" + ''; + + postFixup = lib.optionalString cudaSupport '' + find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do + addOpenGLRunpath "$lib" + done + ''; + }; + + meta = with stdenv.lib; { + description = "Computation using data flow graphs for scalable machine learning"; + homepage = http://tensorflow.org; + license = licenses.asl20; + maintainers = with maintainers; [ jyp abbradar ]; + platforms = with platforms; linux ++ darwin; + # The py2 build fails due to some issue importing protobuf. Possibly related to the fix in + # https://github.com/akesandgren/easybuild-easyblocks/commit/1f2e517ddfd1b00a342c6abb55aef3fd93671a2b + broken = !(xlaSupport -> cudaSupport) || !isPy3k; + }; + }; + +in buildPythonPackage { + inherit version pname; + disabled = isPy27 || (pythonAtLeast "3.8"); + + src = bazel-build.python; + + # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow + # and the propagated input tensorflow-tensorboard, which causes environment collisions. + # Another possibility would be to have tensorboard only in the buildInputs + # https://github.com/tensorflow/tensorflow/blob/v1.7.1/tensorflow/tools/pip_package/setup.py#L79 + postInstall = '' + rm $out/bin/tensorboard + ''; + + setupPyGlobalFlags = [ "--project_name ${pname}" ]; + + # tensorflow/tools/pip_package/setup.py + propagatedBuildInputs = [ + absl-py + astor + gast + google-pasta + keras-applications + keras-preprocessing + numpy + six + protobuf + tensorflow-estimator_1 + termcolor + wrapt + grpcio + opt-einsum + ] ++ lib.optionals (!isPy3k) [ + mock + future + functools32 + ] ++ lib.optionals (pythonOlder "3.4") [ + backports_weakref enum34 + ] ++ lib.optionals withTensorboard [ + tensorflow-tensorboard_1 + ]; + + nativeBuildInputs = lib.optional cudaSupport addOpenGLRunpath; + + postFixup = lib.optionalString cudaSupport '' + find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do + addOpenGLRunpath "$lib" + done + ''; + + # Actual tests are slow and impure. + # TODO try to run them anyway + # TODO better test (files in tensorflow/tools/ci_build/builds/*test) + checkPhase = '' + ${python.interpreter} <= 1.1.6;python_version<"3.4"', + # functools comes with python3, need to install the backport for python2 + 'functools32 >= 3.2.3;python_version<"3"', +- 'gast == 0.2.2', ++ 'gast >= 0.2.2', + 'google_pasta >= 0.1.6', + 'keras_applications >= 1.0.8', + 'keras_preprocessing >= 1.0.5', diff --git a/pkgs/development/python-modules/tensorflow/1/prefetcher.sh b/pkgs/development/python-modules/tensorflow/1/prefetcher.sh new file mode 100755 index 000000000000..d590fb0f1732 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/1/prefetcher.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +version=1.14.0 +hashfile=binary-hashes.nix +rm -f $hashfile +echo "{" >> $hashfile +echo "version = \"$version\";" >> $hashfile +for sys in "linux" "mac"; do + for tfpref in "cpu/tensorflow" "gpu/tensorflow_gpu"; do + for pykind in "py2-none-any" "py3-none-any" "cp27-none-linux_x86_64" "cp35-cp35m-linux_x86_64" "cp36-cp36m-linux_x86_64" "cp37-cp37m-linux_x86_64"; do + if [ $sys == "mac" ]; then + [[ $pykind =~ py.* ]] && [[ $tfpref =~ cpu.* ]] + result=$? + pyver=${pykind:2:1} + flavour=cpu + else + [[ $pykind =~ .*linux.* ]] + result=$? + pyver=${pykind:2:2} + flavour=${tfpref:0:3} + fi + if [ $result == 0 ]; then + url=https://storage.googleapis.com/tensorflow/$sys/$tfpref-$version-$pykind.whl + hash=$(nix-prefetch-url $url) + echo "${sys}_py_${pyver}_${flavour} = {" >> $hashfile + echo " url = \"$url\";" >> $hashfile + echo " sha256 = \"$hash\";" >> $hashfile + echo "};" >> $hashfile + fi + done + done +done +echo "}" >> $hashfile diff --git a/pkgs/development/python-modules/tensorflow/2/bin.nix b/pkgs/development/python-modules/tensorflow/2/bin.nix new file mode 100644 index 000000000000..95a7b2a37485 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/2/bin.nix @@ -0,0 +1,179 @@ +{ stdenv +, lib +, fetchurl +, buildPythonPackage +, isPy3k, pythonOlder, isPy38 +, astor +, gast +, google-pasta +, wrapt +, numpy +, six +, termcolor +, protobuf +, absl-py +, grpcio +, mock +, scipy +, wheel +, opt-einsum +, backports_weakref +, tensorflow-estimator_2 +, tensorflow-tensorboard +, cudaSupport ? false +, cudatoolkit ? null +, cudnn ? null +, nvidia_x11 ? null +, zlib +, python +, symlinkJoin +, keras-applications +, keras-preprocessing +, addOpenGLRunpath +}: + +# We keep this binary build for two reasons: +# - the source build doesn't work on Darwin. +# - the source build is currently brittle and not easy to maintain + +assert cudaSupport -> cudatoolkit != null + && cudnn != null + && nvidia_x11 != null; + +# unsupported combination +assert ! (stdenv.isDarwin && cudaSupport); + +let + packages = import ./binary-hashes.nix; + + variant = if cudaSupport then "-gpu" else ""; + pname = "tensorflow${variant}"; + +in buildPythonPackage { + inherit pname; + inherit (packages) version; + format = "wheel"; + + disabled = isPy38; + + src = let + pyVerNoDot = lib.strings.stringAsChars (x: if x == "." then "" else x) python.pythonVersion; + platform = if stdenv.isDarwin then "mac" else "linux"; + unit = if cudaSupport then "gpu" else "cpu"; + key = "${platform}_py_${pyVerNoDot}_${unit}"; + in fetchurl packages.${key}; + + propagatedBuildInputs = [ + protobuf + numpy + scipy + termcolor + grpcio + six + astor + absl-py + gast + opt-einsum + google-pasta + wrapt + tensorflow-estimator_2 + tensorflow-tensorboard + keras-applications + keras-preprocessing + ] ++ lib.optional (!isPy3k) mock + ++ lib.optionals (pythonOlder "3.4") [ backports_weakref ]; + + nativeBuildInputs = [ wheel ] ++ lib.optional cudaSupport addOpenGLRunpath; + + preConfigure = '' + unset SOURCE_DATE_EPOCH + + # Make sure that dist and the wheel file are writable. + chmod u+rwx -R ./dist + + pushd dist + + # Unpack the wheel file. + wheel unpack --dest unpacked ./*.whl + + # Tensorflow has a hard dependency on gast==0.2.2, but we relax it to + # gast==0.3.2. + substituteInPlace ./unpacked/tensorflow*/tensorflow_core/tools/pip_package/setup.py --replace "gast == 0.2.2" "gast == 0.3.2" + substituteInPlace ./unpacked/tensorflow*/tensorflow_*.dist-info/METADATA --replace "gast (==0.2.2)" "gast (==0.3.2)" + + # Pack the wheel file back up. + wheel pack ./unpacked/tensorflow* + + popd + ''; + + # Note that we need to run *after* the fixup phase because the + # libraries are loaded at runtime. If we run in preFixup then + # patchelf --shrink-rpath will remove the cuda libraries. + postFixup = + let + # rpaths we only need to add if CUDA is enabled. + cudapaths = lib.optionals cudaSupport [ + cudatoolkit.out + cudatoolkit.lib + cudnn + nvidia_x11 + ]; + + libpaths = [ + stdenv.cc.cc.lib + zlib + ]; + + rpath = stdenv.lib.makeLibraryPath (libpaths ++ cudapaths); + in + lib.optionalString stdenv.isLinux '' + # This is an array containing all the directories in the tensorflow2 + # package that contain .so files. + # + # TODO: Create this list programmatically, and remove paths that aren't + # actually needed. + rrPathArr=( + "$out/${python.sitePackages}/tensorflow_core/" + "$out/${python.sitePackages}/tensorflow_core/compiler/tf2tensorrt/" + "$out/${python.sitePackages}/tensorflow_core/compiler/tf2xla/ops/" + "$out/${python.sitePackages}/tensorflow_core/lite/experimental/microfrontend/python/ops/" + "$out/${python.sitePackages}/tensorflow_core/lite/python/interpreter_wrapper/" + "$out/${python.sitePackages}/tensorflow_core/lite/python/optimize/" + "$out/${python.sitePackages}/tensorflow_core/python/" + "$out/${python.sitePackages}/tensorflow_core/python/framework/" + "${rpath}" + ) + + # The the bash array into a colon-separated list of RPATHs. + rrPath=$(IFS=$':'; echo "''${rrPathArr[*]}") + echo "about to run patchelf with the following rpath: $rrPath" + + find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do + echo "about to patchelf $lib..." + chmod a+rx "$lib" + patchelf --set-rpath "$rrPath" "$lib" + ${lib.optionalString cudaSupport '' + addOpenGLRunpath "$lib" + ''} + done + ''; + + pythonImportsCheck = [ + "tensorflow" + "tensorflow.keras" + "tensorflow.python" + "tensorflow.python.framework" + ]; + + meta = with stdenv.lib; { + description = "Computation using data flow graphs for scalable machine learning"; + homepage = http://tensorflow.org; + license = licenses.asl20; + maintainers = with maintainers; [ jyp abbradar cdepillabout ]; + platforms = [ "x86_64-linux" "x86_64-darwin" ]; + # Python 2.7 build uses different string encoding. + # See https://github.com/NixOS/nixpkgs/pull/37044#issuecomment-373452253 + broken = stdenv.isDarwin && !isPy3k; + }; +} diff --git a/pkgs/development/python-modules/tensorflow/2/binary-hashes.nix b/pkgs/development/python-modules/tensorflow/2/binary-hashes.nix new file mode 100644 index 000000000000..90848e93273f --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/2/binary-hashes.nix @@ -0,0 +1,51 @@ +{ +version = "2.1.0"; +linux_py_27_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.1.0-cp27-cp27mu-manylinux2010_x86_64.whl"; + sha256 = "17lnhr7vdrls68c79n3sah5rpd0q1x2v5m84azvlyxxh2wpypfmb"; +}; +linux_py_27_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp27-cp27mu-manylinux2010_x86_64.whl"; + sha256 = "10lz3i4pcpgqrcbjmxm0n7k1gsqlpna3kdid902j2fy060cpi93z"; +}; +linux_py_35_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.1.0-cp35-cp35m-manylinux2010_x86_64.whl"; + sha256 = "09s081n08dpmflwgir3zwzfijfpmahbh2gy5fn5bv5ll86g1szsy"; +}; +linux_py_35_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp35-cp35m-manylinux2010_x86_64.whl"; + sha256 = "1aa7v9fnvx03hqvhl3x3xcn41qy6qxw5xybg54ifjvvicp455c8l"; +}; +linux_py_36_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl"; + sha256 = "1dqp080ljbl9v3115vjp63ls0fimiwym6zxyanyhrlk8kwsq20zc"; +}; +linux_py_36_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl"; + sha256 = "133z8anx7xm9rr5i9s9dwnp1wf06nr6s7q1lbs4lxpk6kn9nl480"; +}; +linux_py_37_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.1.0-cp37-cp37m-manylinux2010_x86_64.whl"; + sha256 = "0yabl3xmcpr67w0zksqs3qc68nl9ax0vcd7w7b35nq8f65xl0ghy"; +}; +linux_py_37_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp37-cp37m-manylinux2010_x86_64.whl"; + sha256 = "04gngbngyg7p1gwx1q89my0cl8j7lq4kknqh51s2ynrix71zvsy6"; +}; +mac_py_27_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.1.0-cp27-cp27m-macosx_10_9_x86_64.whl"; + sha256 = "1mprp72w5kk0lyjm2mh4lf57827xk3wsg28c4gizwm00ydfgacg6"; +}; +mac_py_35_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.1.0-cp35-cp35m-macosx_10_6_intel.whl"; + sha256 = "1as7brf5ai6r7v1di9646jfrbnirpk2b0d1g29mn3shavb62kw8w"; +}; +mac_py_36_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.1.0-cp36-cp36m-macosx_10_9_x86_64.whl"; + sha256 = "1v1rw9kjrskhcq1yas4ly2yfnzf2i1pjh6qg6zixfbkpkw7sw3wc"; +}; +mac_py_37_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.1.0-cp37-cp37m-macosx_10_9_x86_64.whl"; + sha256 = "1hh4n0d97mrq35cmmsrnlmcv9vlswsyjy368lj3pda3y9dvck3rf"; +}; +} diff --git a/pkgs/development/python-modules/tensorflow/2/default.nix b/pkgs/development/python-modules/tensorflow/2/default.nix new file mode 100644 index 000000000000..294303230847 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/2/default.nix @@ -0,0 +1,421 @@ +{ stdenv, pkgs, bazel_0_29, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin +, addOpenGLRunpath +# Python deps +, buildPythonPackage, isPy3k, isPy27, pythonOlder, pythonAtLeast, python +# Python libraries +, numpy, tensorflow-tensorboard_2, backports_weakref, mock, enum34, absl-py +, future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta +, functools32 +, opt-einsum +, termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator_2 +# Common deps +, git, swig, which, binutils, glibcLocales, cython +# Common libraries +, jemalloc, openmpi, astor, gast, grpc, sqlite, openssl, jsoncpp, re2 +, curl, snappy, flatbuffers, icu, double-conversion, libpng, libjpeg, giflib +# Upsteam by default includes cuda support since tensorflow 1.15. We could do +# that in nix as well. It would make some things easier and less confusing, but +# it would also make the default tensorflow package unfree. See +# https://groups.google.com/a/tensorflow.org/forum/#!topic/developers/iRCt5m4qUz0 +, cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null, nccl ? null +# XLA without CUDA is broken +, xlaSupport ? cudaSupport +# Default from ./configure script +, cudaCapabilities ? [ "3.5" "5.2" ] +, sse42Support ? builtins.elem (stdenv.hostPlatform.platform.gcc.arch or "default") ["westmere" "sandybridge" "ivybridge" "haswell" "broadwell" "skylake" "skylake-avx512"] +, avx2Support ? builtins.elem (stdenv.hostPlatform.platform.gcc.arch or "default") [ "haswell" "broadwell" "skylake" "skylake-avx512"] +, fmaSupport ? builtins.elem (stdenv.hostPlatform.platform.gcc.arch or "default") [ "haswell" "broadwell" "skylake" "skylake-avx512"] +# Darwin deps +, Foundation, Security +}: + +assert cudaSupport -> nvidia_x11 != null + && cudatoolkit != null + && cudnn != null; + +# unsupported combination +assert ! (stdenv.isDarwin && cudaSupport); + +let + withTensorboard = pythonOlder "3.6"; + + cudatoolkit_joined = symlinkJoin { + name = "${cudatoolkit.name}-merged"; + paths = [ + cudatoolkit.lib + cudatoolkit.out + # for some reason some of the required libs are in the targets/x86_64-linux + # directory; not sure why but this works around it + "${cudatoolkit}/targets/${stdenv.system}" + ]; + }; + + cudatoolkit_cc_joined = symlinkJoin { + name = "${cudatoolkit.cc.name}-merged"; + paths = [ + cudatoolkit.cc + binutils.bintools # for ar, dwp, nm, objcopy, objdump, strip + ]; + }; + + # Needed for _some_ system libraries, grep INCLUDEDIR. + includes_joined = symlinkJoin { + name = "tensorflow-deps-merged"; + paths = [ + pkgs.protobuf + jsoncpp + ]; + }; + + tfFeature = x: if x then "1" else "0"; + + version = "2.1.0"; + variant = if cudaSupport then "-gpu" else ""; + pname = "tensorflow${variant}"; + + pythonEnv = python.withPackages (_: + [ # python deps needed during wheel build time (not runtime, see the buildPythonPackage part for that) + numpy + keras-preprocessing + protobuf + wrapt + gast + astor + absl-py + termcolor + keras-applications + setuptools + wheel + ] ++ lib.optionals (!isPy3k) + [ future + functools32 + mock + ]); + + bazel-build = buildBazelPackage { + name = "${pname}-${version}"; + bazel = bazel_0_29; + + src = fetchFromGitHub { + owner = "tensorflow"; + repo = "tensorflow"; + rev = "v${version}"; + sha256 = "1g79xi8yl4sjia8ysk9b7xfzrz83zy28v5dlb2wzmcf0k5pmz60p"; + }; + + patches = [ + # Work around https://github.com/tensorflow/tensorflow/issues/24752 + ../no-saved-proto.patch + # Fixes for NixOS jsoncpp + ../system-jsoncpp.patch + + (fetchpatch { + name = "backport-pr-18950.patch"; + url = "https://github.com/tensorflow/tensorflow/commit/73640aaec2ab0234d9fff138e3c9833695570c0a.patch"; + sha256 = "1n9ypbrx36fc1kc9cz5b3p9qhg15xxhq4nz6ap3hwqba535nakfz"; + }) + + (fetchpatch { + # Don't try to fetch things that don't exist + name = "prune-missing-deps.patch"; + url = "https://github.com/tensorflow/tensorflow/commit/b39b1ed24b4814db27d2f748dc85c10730ae851d.patch"; + sha256 = "1skysz53nancvw1slij6s7flar2kv3gngnsq60ff4lap88kx5s6c"; + excludes = [ "tensorflow/cc/saved_model/BUILD" ]; + }) + + ./lift-gast-restriction.patch + + # cuda 10.2 does not have "-bin2c-path" option anymore + # https://github.com/tensorflow/tensorflow/issues/34429 + ../cuda-10.2-no-bin2c-path.patch + ]; + + # On update, it can be useful to steal the changes from gentoo + # https://gitweb.gentoo.org/repo/gentoo.git/tree/sci-libs/tensorflow + + nativeBuildInputs = [ + swig which pythonEnv + ] ++ lib.optional cudaSupport addOpenGLRunpath; + + buildInputs = [ + jemalloc + openmpi + glibcLocales + git + + # libs taken from system through the TF_SYS_LIBS mechanism + # grpc + sqlite + openssl + jsoncpp + pkgs.protobuf + curl + snappy + flatbuffers + icu + double-conversion + libpng + libjpeg + giflib + re2 + pkgs.lmdb + ] ++ lib.optionals cudaSupport [ + cudatoolkit + cudnn + nvidia_x11 + ] ++ lib.optionals stdenv.isDarwin [ + Foundation + Security + ]; + + # arbitrarily set to the current latest bazel version, overly careful + TF_IGNORE_MAX_BAZEL_VERSION = true; + + # Take as many libraries from the system as possible. Keep in sync with + # list of valid syslibs in + # https://github.com/tensorflow/tensorflow/blob/master/third_party/systemlibs/syslibs_configure.bzl + TF_SYSTEM_LIBS = lib.concatStringsSep "," [ + "absl_py" + "astor_archive" + "boringssl" + # Not packaged in nixpkgs + # "com_github_googleapis_googleapis" + # "com_github_googlecloudplatform_google_cloud_cpp" + "com_google_protobuf" + "com_googlesource_code_re2" + "curl" + "cython" + "double_conversion" + "flatbuffers" + "gast_archive" + # Lots of errors, requires an older version + # "grpc" + "hwloc" + "icu" + "jpeg" + "jsoncpp_git" + "keras_applications_archive" + "lmdb" + "nasm" + # "nsync" # not packaged in nixpkgs + "opt_einsum_archive" + "org_sqlite" + "pasta" + "pcre" + "six_archive" + "snappy" + "swig" + "termcolor_archive" + "wrapt" + "zlib_archive" + ]; + + INCLUDEDIR = "${includes_joined}/include"; + + PYTHON_BIN_PATH = pythonEnv.interpreter; + + TF_NEED_GCP = true; + TF_NEED_HDFS = true; + TF_ENABLE_XLA = tfFeature xlaSupport; + + CC_OPT_FLAGS = " "; + + # https://github.com/tensorflow/tensorflow/issues/14454 + TF_NEED_MPI = tfFeature cudaSupport; + + TF_NEED_CUDA = tfFeature cudaSupport; + TF_CUDA_PATHS = lib.optionalString cudaSupport "${cudatoolkit_joined},${cudnn},${nccl}"; + GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin"; + GCC_HOST_COMPILER_PATH = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin/gcc"; + TF_CUDA_COMPUTE_CAPABILITIES = lib.concatStringsSep "," cudaCapabilities; + + postPatch = '' + # https://github.com/tensorflow/tensorflow/issues/20919 + sed -i '/androidndk/d' tensorflow/lite/kernels/internal/BUILD + + # Tensorboard pulls in a bunch of dependencies, some of which may + # include security vulnerabilities. So we make it optional. + # https://github.com/tensorflow/tensorflow/issues/20280#issuecomment-400230560 + sed -i '/tensorboard >=/d' tensorflow/tools/pip_package/setup.py + ''; + + preConfigure = let + opt_flags = [] + ++ lib.optionals sse42Support ["-msse4.2"] + ++ lib.optionals avx2Support ["-mavx2"] + ++ lib.optionals fmaSupport ["-mfma"]; + in '' + patchShebangs configure + + # dummy ldconfig + mkdir dummy-ldconfig + echo "#!${stdenv.shell}" > dummy-ldconfig/ldconfig + chmod +x dummy-ldconfig/ldconfig + export PATH="$PWD/dummy-ldconfig:$PATH" + + export PYTHON_LIB_PATH="$NIX_BUILD_TOP/site-packages" + export CC_OPT_FLAGS="${lib.concatStringsSep " " opt_flags}" + mkdir -p "$PYTHON_LIB_PATH" + + # To avoid mixing Python 2 and Python 3 + unset PYTHONPATH + ''; + + configurePhase = '' + runHook preConfigure + ./configure + runHook postConfigure + ''; + + # FIXME: Tensorflow uses dlopen() for CUDA libraries. + NIX_LDFLAGS = lib.optionalString cudaSupport "-lcudart -lcublas -lcufft -lcurand -lcusolver -lcusparse -lcudnn"; + + hardeningDisable = [ "format" ]; + + bazelFlags = [ + # temporary fixes to make the build work with bazel 0.27 + "--incompatible_no_support_tools_in_action_inputs=false" + ]; + bazelBuildFlags = [ + "--config=opt" # optimize using the flags set in the configure phase + ]; + + bazelTarget = "//tensorflow/tools/pip_package:build_pip_package //tensorflow/tools/lib_package:libtensorflow"; + + fetchAttrs = { + # So that checksums don't depend on these. + TF_SYSTEM_LIBS = null; + + # cudaSupport causes fetch of ncclArchive, resulting in different hashes + sha256 = if cudaSupport then + "0hg3ysy644950a34j28hrb317cz8gcbb9n84d36wdailvnlshghb" + else + "1gy4pz9kn30wb9c4a9584fibb88c3h38y3dqa99yw1lbsbyyi28c"; + }; + + buildAttrs = { + outputs = [ "out" "python" ]; + + preBuild = '' + patchShebangs . + ''; + + installPhase = '' + mkdir -p "$out" + tar -xf bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz -C "$out" + # Write pkgconfig file. + mkdir "$out/lib/pkgconfig" + cat > "$out/lib/pkgconfig/tensorflow.pc" << EOF + Name: TensorFlow + Version: ${version} + Description: Library for computation using data flow graphs for scalable machine learning + Requires: + Libs: -L$out/lib -ltensorflow + Cflags: -I$out/include/tensorflow + EOF + + # build the source code, then copy it to $python (build_pip_package + # actually builds a symlink farm so we must dereference them). + bazel-bin/tensorflow/tools/pip_package/build_pip_package --src "$PWD/dist" + cp -Lr "$PWD/dist" "$python" + ''; + + postFixup = lib.optionalString cudaSupport '' + find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do + addOpenGLRunpath "$lib" + done + ''; + }; + + meta = with stdenv.lib; { + description = "Computation using data flow graphs for scalable machine learning"; + homepage = http://tensorflow.org; + license = licenses.asl20; + maintainers = with maintainers; [ jyp abbradar ]; + platforms = with platforms; linux ++ darwin; + # The py2 build fails due to some issue importing protobuf. Possibly related to the fix in + # https://github.com/akesandgren/easybuild-easyblocks/commit/1f2e517ddfd1b00a342c6abb55aef3fd93671a2b + broken = !(xlaSupport -> cudaSupport) || !isPy3k; + }; + }; + +in buildPythonPackage { + inherit version pname; + disabled = isPy27 || (pythonAtLeast "3.8"); + + src = bazel-build.python; + + # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow + # and the propagated input tensorflow-tensorboard, which causes environment collisions. + # Another possibility would be to have tensorboard only in the buildInputs + # https://github.com/tensorflow/tensorflow/blob/v1.7.1/tensorflow/tools/pip_package/setup.py#L79 + postInstall = '' + rm $out/bin/tensorboard + ''; + + setupPyGlobalFlags = [ "--project_name ${pname}" ]; + + # tensorflow/tools/pip_package/setup.py + propagatedBuildInputs = [ + absl-py + astor + gast + google-pasta + keras-applications + keras-preprocessing + numpy + six + protobuf + tensorflow-estimator_2 + termcolor + wrapt + grpcio + opt-einsum + ] ++ lib.optionals (!isPy3k) [ + mock + future + functools32 + ] ++ lib.optionals (pythonOlder "3.4") [ + backports_weakref enum34 + ] ++ lib.optionals withTensorboard [ + tensorflow-tensorboard_2 + ]; + + nativeBuildInputs = lib.optional cudaSupport addOpenGLRunpath; + + postFixup = lib.optionalString cudaSupport '' + find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do + addOpenGLRunpath "$lib" + done + ''; + + # Actual tests are slow and impure. + # TODO try to run them anyway + # TODO better test (files in tensorflow/tools/ci_build/builds/*test) + checkPhase = '' + ${python.interpreter} <= 1.1.6;python_version<"3.4"', +- 'gast == 0.2.2', ++ 'gast >= 0.2.2', + 'google_pasta >= 0.1.6', + 'keras_applications >= 1.0.8', + 'keras_preprocessing >= 1.0.5', diff --git a/pkgs/development/python-modules/tensorflow/2/prefetcher.sh b/pkgs/development/python-modules/tensorflow/2/prefetcher.sh new file mode 100755 index 000000000000..abb0faac2834 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/2/prefetcher.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +version=2.1.0 + +# List of binary wheels for Tensorflow. The most recent versions can be found +# on the following page: +# https://www.tensorflow.org/install/pip?lang=python3#package-location +url_and_key_list=( + "linux_py_27_gpu https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${version}-cp27-cp27mu-manylinux2010_x86_64.whl" + "linux_py_27_cpu https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-${version}-cp27-cp27mu-manylinux2010_x86_64.whl" + "linux_py_35_gpu https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${version}-cp35-cp35m-manylinux2010_x86_64.whl" + "linux_py_35_cpu https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-${version}-cp35-cp35m-manylinux2010_x86_64.whl" + "linux_py_36_gpu https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${version}-cp36-cp36m-manylinux2010_x86_64.whl" + "linux_py_36_cpu https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-${version}-cp36-cp36m-manylinux2010_x86_64.whl" + "linux_py_37_gpu https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${version}-cp37-cp37m-manylinux2010_x86_64.whl" + "linux_py_37_cpu https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-${version}-cp37-cp37m-manylinux2010_x86_64.whl" + "mac_py_27_cpu https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-cp27-cp27m-macosx_10_9_x86_64.whl" + "mac_py_35_cpu https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-cp35-cp35m-macosx_10_6_intel.whl" + "mac_py_36_cpu https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-cp36-cp36m-macosx_10_9_x86_64.whl" + "mac_py_37_cpu https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-cp37-cp37m-macosx_10_9_x86_64.whl" +) + +hashfile=binary-hashes.nix +rm -f $hashfile +echo "{" >> $hashfile +echo "version = \"$version\";" >> $hashfile + +for url_and_key in "${url_and_key_list[@]}"; do + key=$(echo "$url_and_key" | cut -d' ' -f1) + url=$(echo "$url_and_key" | cut -d' ' -f2) + + echo "prefetching ${url}..." + hash=$(nix-prefetch-url $url) + + echo "$key = {" >> $hashfile + echo " url = \"$url\";" >> $hashfile + echo " sha256 = \"$hash\";" >> $hashfile + echo "};" >> $hashfile + + echo +done + +echo "}" >> $hashfile +echo "done." diff --git a/pkgs/development/python-modules/tensorflow/bin.nix b/pkgs/development/python-modules/tensorflow/bin.nix deleted file mode 100644 index 38fec9adc7dc..000000000000 --- a/pkgs/development/python-modules/tensorflow/bin.nix +++ /dev/null @@ -1,121 +0,0 @@ -{ stdenv -, lib -, fetchurl -, buildPythonPackage -, isPy3k, pythonOlder, isPy38 -, astor -, gast -, google-pasta -, wrapt -, numpy -, six -, termcolor -, protobuf -, absl-py -, grpcio -, mock -, backports_weakref -, tensorflow-estimator -, tensorflow-tensorboard -, cudaSupport ? false -, cudatoolkit ? null -, cudnn ? null -, nvidia_x11 ? null -, zlib -, python -, symlinkJoin -, keras-applications -, keras-preprocessing -, addOpenGLRunpath -}: - -# We keep this binary build for two reasons: -# - the source build doesn't work on Darwin. -# - the source build is currently brittle and not easy to maintain - -assert cudaSupport -> cudatoolkit != null - && cudnn != null - && nvidia_x11 != null; - -# unsupported combination -assert ! (stdenv.isDarwin && cudaSupport); - -let - packages = import ./binary-hashes.nix; - - variant = if cudaSupport then "-gpu" else ""; - pname = "tensorflow${variant}"; - -in buildPythonPackage { - inherit pname; - inherit (packages) version; - format = "wheel"; - - disabled = isPy38; - - src = let - pyVerNoDot = lib.strings.stringAsChars (x: if x == "." then "" else x) python.pythonVersion; - pyver = if stdenv.isDarwin then builtins.substring 0 1 pyVerNoDot else pyVerNoDot; - platform = if stdenv.isDarwin then "mac" else "linux"; - unit = if cudaSupport then "gpu" else "cpu"; - key = "${platform}_py_${pyver}_${unit}"; - in fetchurl packages.${key}; - - propagatedBuildInputs = [ - protobuf - numpy - termcolor - grpcio - six - astor - absl-py - gast - google-pasta - wrapt - tensorflow-estimator - tensorflow-tensorboard - keras-applications - keras-preprocessing - ] ++ lib.optional (!isPy3k) mock - ++ lib.optionals (pythonOlder "3.4") [ backports_weakref ]; - - nativeBuildInputs = lib.optional cudaSupport addOpenGLRunpath; - - # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow - # and the propageted input tensorflow-tensorboard which causes environment collisions. - # another possibility would be to have tensorboard only in the buildInputs - # https://github.com/tensorflow/tensorflow/blob/v1.7.1/tensorflow/tools/pip_package/setup.py#L79 - postInstall = '' - rm $out/bin/tensorboard - ''; - - # Note that we need to run *after* the fixup phase because the - # libraries are loaded at runtime. If we run in preFixup then - # patchelf --shrink-rpath will remove the cuda libraries. - postFixup = let - rpath = stdenv.lib.makeLibraryPath - ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]); - in - lib.optionalString stdenv.isLinux '' - rrPath="$out/${python.sitePackages}/tensorflow/:$out/${python.sitePackages}/tensorflow/contrib/tensor_forest/:${rpath}" - internalLibPath="$out/${python.sitePackages}/tensorflow/python/_pywrap_tensorflow_internal.so" - find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do - patchelf --set-rpath "$rrPath" "$lib" - ${lib.optionalString cudaSupport '' - addOpenGLRunpath "$lib" - ''} - done - ''; - - - meta = with stdenv.lib; { - description = "Computation using data flow graphs for scalable machine learning"; - homepage = http://tensorflow.org; - license = licenses.asl20; - maintainers = with maintainers; [ jyp abbradar ]; - platforms = [ "x86_64-linux" "x86_64-darwin" ]; - # Python 2.7 build uses different string encoding. - # See https://github.com/NixOS/nixpkgs/pull/37044#issuecomment-373452253 - broken = stdenv.isDarwin && !isPy3k; - }; -} diff --git a/pkgs/development/python-modules/tensorflow/binary-hashes.nix b/pkgs/development/python-modules/tensorflow/binary-hashes.nix deleted file mode 100644 index fa4809dc3f19..000000000000 --- a/pkgs/development/python-modules/tensorflow/binary-hashes.nix +++ /dev/null @@ -1,43 +0,0 @@ -{ -version = "1.14.0"; -linux_py_27_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp27-none-linux_x86_64.whl"; - sha256 = "0yywdrfk97dh1bxhibspg0raz70fx9lcczj6xlimqy4xb60clx7k"; -}; -linux_py_35_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp35-cp35m-linux_x86_64.whl"; - sha256 = "1xvyb6xcrjhlwvrmrhn5vs9xy7g98smqmpv4i3hhpry4qyasphhj"; -}; -linux_py_36_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl"; - sha256 = "1psd9vyxz9f39dwj77nvrg373sxv3p5vdp9fnz81dpsm0b0mwl44"; -}; -linux_py_37_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp37-cp37m-linux_x86_64.whl"; - sha256 = "0bg2sb1n2ag27r7ww695kg5hb0mjrw4kc5893krmixx2j71860c5"; -}; -linux_py_27_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp27-none-linux_x86_64.whl"; - sha256 = "0y1x91gayg6pjddgl8ndcm63wfzhyv4s5khgl7ffzsgni1ivaqw5"; -}; -linux_py_35_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp35-cp35m-linux_x86_64.whl"; - sha256 = "03piggpbz1jx8m2b95spq3jrdff4w6xx63ji07am7hyw2nsgx3mx"; -}; -linux_py_36_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp36-cp36m-linux_x86_64.whl"; - sha256 = "0ypkp8cfhharsyyikb1qgf44cfm6284km9xswzvzymjzz75vg3gd"; -}; -linux_py_37_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp37-cp37m-linux_x86_64.whl"; - sha256 = "0virp8nn2ysx4855hq29kas6fm6b3dsiybwzdxy9nnb9n2d8qlm2"; -}; -mac_py_2_cpu = { - url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py2-none-any.whl"; - sha256 = "14f86k3pgq7z6i4s4im55zpp38f0drnm7xlclavsgcc0nxnj3z26"; -}; -mac_py_3_cpu = { - url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py3-none-any.whl"; - sha256 = "0f3swpcjfgqhj6h5wnx8snc0xjkx4hnkqx83fmlrwpncs8c131d3"; -}; -} diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix deleted file mode 100644 index eafc5ba7447a..000000000000 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ /dev/null @@ -1,431 +0,0 @@ -{ stdenv, pkgs, bazel_0_26, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin -, addOpenGLRunpath -# Python deps -, buildPythonPackage, isPy3k, isPy27, pythonOlder, pythonAtLeast, python -# Python libraries -, numpy, tensorflow-tensorboard, backports_weakref, mock, enum34, absl-py -, future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta -, functools32 -, opt-einsum -, termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator_1_15_1 -# Common deps -, git, swig, which, binutils, glibcLocales, cython -# Common libraries -, jemalloc, openmpi, astor, gast, grpc, sqlite, openssl, jsoncpp, re2 -, curl, snappy, flatbuffers, icu, double-conversion, libpng, libjpeg, giflib -# Upsteam by default includes cuda support since tensorflow 1.15. We could do -# that in nix as well. It would make some things easier and less confusing, but -# it would also make the default tensorflow package unfree. See -# https://groups.google.com/a/tensorflow.org/forum/#!topic/developers/iRCt5m4qUz0 -, cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null, nccl ? null -# XLA without CUDA is broken -, xlaSupport ? cudaSupport -# Default from ./configure script -, cudaCapabilities ? [ "3.5" "5.2" ] -, sse42Support ? builtins.elem (stdenv.hostPlatform.platform.gcc.arch or "default") ["westmere" "sandybridge" "ivybridge" "haswell" "broadwell" "skylake" "skylake-avx512"] -, avx2Support ? builtins.elem (stdenv.hostPlatform.platform.gcc.arch or "default") [ "haswell" "broadwell" "skylake" "skylake-avx512"] -, fmaSupport ? builtins.elem (stdenv.hostPlatform.platform.gcc.arch or "default") [ "haswell" "broadwell" "skylake" "skylake-avx512"] -# Darwin deps -, Foundation, Security -}: - -assert cudaSupport -> nvidia_x11 != null - && cudatoolkit != null - && cudnn != null; - -# unsupported combination -assert ! (stdenv.isDarwin && cudaSupport); - -let - withTensorboard = pythonOlder "3.6"; - - cudatoolkit_joined = symlinkJoin { - name = "${cudatoolkit.name}-merged"; - paths = [ - cudatoolkit.lib - cudatoolkit.out - # for some reason some of the required libs are in the targets/x86_64-linux - # directory; not sure why but this works around it - "${cudatoolkit}/targets/${stdenv.system}" - ]; - }; - - cudatoolkit_cc_joined = symlinkJoin { - name = "${cudatoolkit.cc.name}-merged"; - paths = [ - cudatoolkit.cc - binutils.bintools # for ar, dwp, nm, objcopy, objdump, strip - ]; - }; - - # Needed for _some_ system libraries, grep INCLUDEDIR. - includes_joined = symlinkJoin { - name = "tensorflow-deps-merged"; - paths = [ - pkgs.protobuf - jsoncpp - ]; - }; - - tfFeature = x: if x then "1" else "0"; - - version = "1.15.2"; - variant = if cudaSupport then "-gpu" else ""; - pname = "tensorflow${variant}"; - - pythonEnv = python.withPackages (_: - [ # python deps needed during wheel build time (not runtime, see the buildPythonPackage part for that) - numpy - keras-preprocessing - protobuf - wrapt - gast - astor - absl-py - termcolor - keras-applications - setuptools - wheel - ] ++ lib.optionals (!isPy3k) - [ future - functools32 - mock - ]); - - bazel-build = buildBazelPackage { - name = "${pname}-${version}"; - bazel = bazel_0_26; - - src = fetchFromGitHub { - owner = "tensorflow"; - repo = "tensorflow"; - rev = "v${version}"; - sha256 = "1q0848drjvnaaa38dgns8knmpmkj5plzsc98j20m5ybv68s55w78"; - }; - - patches = [ - # Work around https://github.com/tensorflow/tensorflow/issues/24752 - ./no-saved-proto.patch - # Fixes for NixOS jsoncpp - ./system-jsoncpp.patch - - # https://github.com/tensorflow/tensorflow/pull/29673 - (fetchpatch { - name = "fix-compile-with-cuda-and-mpi.patch"; - url = "https://github.com/tensorflow/tensorflow/pull/29673/commits/498e35a3bfe38dd75cf1416a1a23c07c3b59e6af.patch"; - sha256 = "1m2qmwv1ysqa61z6255xggwbq6mnxbig749bdvrhnch4zydxb4di"; - }) - (fetchpatch { - name = "backport-pr-18950.patch"; - url = "https://github.com/tensorflow/tensorflow/commit/73640aaec2ab0234d9fff138e3c9833695570c0a.patch"; - sha256 = "1n9ypbrx36fc1kc9cz5b3p9qhg15xxhq4nz6ap3hwqba535nakfz"; - }) - - - (fetchpatch { - # be compatible with gast >0.2 instead of only gast 0.2.2 - name = "gast-update.patch"; - url = "https://github.com/tensorflow/tensorflow/commit/85751ad6c7f5fd12c6c79545d96896cba92fa8b4.patch"; - sha256 = "077cpj0kzyqxzdya1dwh8df17zfzhqn7c685hx6iskvw2979zg2n"; - }) - ./lift-gast-restriction.patch - - # cuda 10.2 does not have "-bin2c-path" option anymore - # https://github.com/tensorflow/tensorflow/issues/34429 - ./cuda-10.2-no-bin2c-path.patch - ]; - - # On update, it can be useful to steal the changes from gentoo - # https://gitweb.gentoo.org/repo/gentoo.git/tree/sci-libs/tensorflow - - nativeBuildInputs = [ - swig which pythonEnv - ] ++ lib.optional cudaSupport addOpenGLRunpath; - - buildInputs = [ - jemalloc - openmpi - glibcLocales - git - - # libs taken from system through the TF_SYS_LIBS mechanism - # grpc - sqlite - openssl - jsoncpp - pkgs.protobuf - curl - snappy - flatbuffers - icu - double-conversion - libpng - libjpeg - giflib - re2 - pkgs.lmdb - ] ++ lib.optionals cudaSupport [ - cudatoolkit - cudnn - nvidia_x11 - ] ++ lib.optionals stdenv.isDarwin [ - Foundation - Security - ]; - - # arbitrarily set to the current latest bazel version, overly careful - TF_IGNORE_MAX_BAZEL_VERSION = true; - - # Take as many libraries from the system as possible. Keep in sync with - # list of valid syslibs in - # https://github.com/tensorflow/tensorflow/blob/master/third_party/systemlibs/syslibs_configure.bzl - TF_SYSTEM_LIBS = lib.concatStringsSep "," [ - "absl_py" - "astor_archive" - "boringssl" - # Not packaged in nixpkgs - # "com_github_googleapis_googleapis" - # "com_github_googlecloudplatform_google_cloud_cpp" - "com_google_protobuf" - "com_googlesource_code_re2" - "curl" - "cython" - "double_conversion" - "flatbuffers" - "gast_archive" - "gif_archive" - # Lots of errors, requires an older version - # "grpc" - "hwloc" - "icu" - "jpeg" - "jsoncpp_git" - "keras_applications_archive" - "lmdb" - "nasm" - # "nsync" # not packaged in nixpkgs - "opt_einsum_archive" - "org_sqlite" - "pasta" - "pcre" - "png_archive" - "six_archive" - "snappy" - "swig" - "termcolor_archive" - "wrapt" - "zlib_archive" - ]; - - INCLUDEDIR = "${includes_joined}/include"; - - PYTHON_BIN_PATH = pythonEnv.interpreter; - - TF_NEED_GCP = true; - TF_NEED_HDFS = true; - TF_ENABLE_XLA = tfFeature xlaSupport; - - CC_OPT_FLAGS = " "; - - # https://github.com/tensorflow/tensorflow/issues/14454 - TF_NEED_MPI = tfFeature cudaSupport; - - TF_NEED_CUDA = tfFeature cudaSupport; - TF_CUDA_PATHS = lib.optionalString cudaSupport "${cudatoolkit_joined},${cudnn},${nccl}"; - GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin"; - GCC_HOST_COMPILER_PATH = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin/gcc"; - TF_CUDA_COMPUTE_CAPABILITIES = lib.concatStringsSep "," cudaCapabilities; - - postPatch = '' - # https://github.com/tensorflow/tensorflow/issues/20919 - sed -i '/androidndk/d' tensorflow/lite/kernels/internal/BUILD - - # Tensorboard pulls in a bunch of dependencies, some of which may - # include security vulnerabilities. So we make it optional. - # https://github.com/tensorflow/tensorflow/issues/20280#issuecomment-400230560 - sed -i '/tensorboard >=/d' tensorflow/tools/pip_package/setup.py - ''; - - preConfigure = let - opt_flags = [] - ++ lib.optionals sse42Support ["-msse4.2"] - ++ lib.optionals avx2Support ["-mavx2"] - ++ lib.optionals fmaSupport ["-mfma"]; - in '' - patchShebangs configure - - # dummy ldconfig - mkdir dummy-ldconfig - echo "#!${stdenv.shell}" > dummy-ldconfig/ldconfig - chmod +x dummy-ldconfig/ldconfig - export PATH="$PWD/dummy-ldconfig:$PATH" - - export PYTHON_LIB_PATH="$NIX_BUILD_TOP/site-packages" - export CC_OPT_FLAGS="${lib.concatStringsSep " " opt_flags}" - mkdir -p "$PYTHON_LIB_PATH" - - # To avoid mixing Python 2 and Python 3 - unset PYTHONPATH - ''; - - configurePhase = '' - runHook preConfigure - ./configure - runHook postConfigure - ''; - - # FIXME: Tensorflow uses dlopen() for CUDA libraries. - NIX_LDFLAGS = lib.optionalString cudaSupport "-lcudart -lcublas -lcufft -lcurand -lcusolver -lcusparse -lcudnn"; - - hardeningDisable = [ "format" ]; - - bazelFlags = [ - # temporary fixes to make the build work with bazel 0.27 - "--incompatible_no_support_tools_in_action_inputs=false" - ]; - bazelBuildFlags = [ - "--config=opt" # optimize using the flags set in the configure phase - ]; - - bazelTarget = "//tensorflow/tools/pip_package:build_pip_package //tensorflow/tools/lib_package:libtensorflow"; - - fetchAttrs = { - # So that checksums don't depend on these. - TF_SYSTEM_LIBS = null; - - # cudaSupport causes fetch of ncclArchive, resulting in different hashes - sha256 = if cudaSupport then - "1qygfcvvn9vysap9nk6xccxi9mgmzyxiywz6k456f811l1v70p2c" - else - "0kfjanw0mfbh30vi1ms2xlg8yp429cbyfriik6yxd5cla2pncg2j"; - }; - - buildAttrs = { - outputs = [ "out" "python" ]; - - preBuild = '' - patchShebangs . - ''; - - installPhase = '' - mkdir -p "$out" - tar -xf bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz -C "$out" - # Write pkgconfig file. - mkdir "$out/lib/pkgconfig" - cat > "$out/lib/pkgconfig/tensorflow.pc" << EOF - Name: TensorFlow - Version: ${version} - Description: Library for computation using data flow graphs for scalable machine learning - Requires: - Libs: -L$out/lib -ltensorflow - Cflags: -I$out/include/tensorflow - EOF - - # build the source code, then copy it to $python (build_pip_package - # actually builds a symlink farm so we must dereference them). - bazel-bin/tensorflow/tools/pip_package/build_pip_package --src "$PWD/dist" - cp -Lr "$PWD/dist" "$python" - ''; - - postFixup = lib.optionalString cudaSupport '' - find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do - addOpenGLRunpath "$lib" - done - ''; - }; - - meta = with stdenv.lib; { - description = "Computation using data flow graphs for scalable machine learning"; - homepage = http://tensorflow.org; - license = licenses.asl20; - maintainers = with maintainers; [ jyp abbradar ]; - platforms = with platforms; linux ++ darwin; - # The py2 build fails due to some issue importing protobuf. Possibly related to the fix in - # https://github.com/akesandgren/easybuild-easyblocks/commit/1f2e517ddfd1b00a342c6abb55aef3fd93671a2b - broken = !(xlaSupport -> cudaSupport) || !isPy3k; - }; - }; - -in buildPythonPackage { - inherit version pname; - disabled = isPy27 || (pythonAtLeast "3.8"); - - src = bazel-build.python; - - # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow - # and the propagated input tensorflow-tensorboard, which causes environment collisions. - # Another possibility would be to have tensorboard only in the buildInputs - # https://github.com/tensorflow/tensorflow/blob/v1.7.1/tensorflow/tools/pip_package/setup.py#L79 - postInstall = '' - rm $out/bin/tensorboard - ''; - - setupPyGlobalFlags = [ "--project_name ${pname}" ]; - - # tensorflow/tools/pip_package/setup.py - propagatedBuildInputs = [ - absl-py - astor - gast - google-pasta - keras-applications - keras-preprocessing - numpy - six - protobuf - tensorflow-estimator_1_15_1 - termcolor - wrapt - grpcio - opt-einsum - ] ++ lib.optionals (!isPy3k) [ - mock - future - functools32 - ] ++ lib.optionals (pythonOlder "3.4") [ - backports_weakref enum34 - ] ++ lib.optionals withTensorboard [ - tensorflow-tensorboard - ]; - - nativeBuildInputs = lib.optional cudaSupport addOpenGLRunpath; - - postFixup = lib.optionalString cudaSupport '' - find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do - addOpenGLRunpath "$lib" - done - ''; - - # Actual tests are slow and impure. - # TODO try to run them anyway - # TODO better test (files in tensorflow/tools/ci_build/builds/*test) - checkPhase = '' - ${python.interpreter} <= 1.1.6;python_version<"3.4"', - # functools comes with python3, need to install the backport for python2 - 'functools32 >= 3.2.3;python_version<"3"', -- 'gast == 0.2.2', -+ 'gast >= 0.2.2', - 'google_pasta >= 0.1.6', - 'keras_applications >= 1.0.8', - 'keras_preprocessing >= 1.0.5', diff --git a/pkgs/development/python-modules/tensorflow/prefetcher.sh b/pkgs/development/python-modules/tensorflow/prefetcher.sh deleted file mode 100755 index d590fb0f1732..000000000000 --- a/pkgs/development/python-modules/tensorflow/prefetcher.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -version=1.14.0 -hashfile=binary-hashes.nix -rm -f $hashfile -echo "{" >> $hashfile -echo "version = \"$version\";" >> $hashfile -for sys in "linux" "mac"; do - for tfpref in "cpu/tensorflow" "gpu/tensorflow_gpu"; do - for pykind in "py2-none-any" "py3-none-any" "cp27-none-linux_x86_64" "cp35-cp35m-linux_x86_64" "cp36-cp36m-linux_x86_64" "cp37-cp37m-linux_x86_64"; do - if [ $sys == "mac" ]; then - [[ $pykind =~ py.* ]] && [[ $tfpref =~ cpu.* ]] - result=$? - pyver=${pykind:2:1} - flavour=cpu - else - [[ $pykind =~ .*linux.* ]] - result=$? - pyver=${pykind:2:2} - flavour=${tfpref:0:3} - fi - if [ $result == 0 ]; then - url=https://storage.googleapis.com/tensorflow/$sys/$tfpref-$version-$pykind.whl - hash=$(nix-prefetch-url $url) - echo "${sys}_py_${pyver}_${flavour} = {" >> $hashfile - echo " url = \"$url\";" >> $hashfile - echo " sha256 = \"$hash\";" >> $hashfile - echo "};" >> $hashfile - fi - done - done -done -echo "}" >> $hashfile diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 648b815e9ea2..7d1892f2be11 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -6548,21 +6548,49 @@ in { zerobin = callPackage ../development/python-modules/zerobin { }; - tensorflow-estimator = callPackage ../development/python-modules/tensorflow-estimator { }; - tensorflow-estimator_1_15_1 = callPackage ../development/python-modules/tensorflow-estimator/1_15_1.nix { }; + tensorflow-estimator = self.tensorflow-estimator_1; + + tensorflow-estimator_1 = callPackage ../development/python-modules/tensorflow-estimator/1 { }; + + tensorflow-estimator_2 = callPackage ../development/python-modules/tensorflow-estimator/2 { }; tensorflow-probability = callPackage ../development/python-modules/tensorflow-probability { }; - tensorflow-tensorboard = callPackage ../development/python-modules/tensorflow-tensorboard { }; + tensorflow-tensorboard = self.tensorflow-tensorboard_1; + + tensorflow-tensorboard_1 = callPackage ../development/python-modules/tensorflow-tensorboard/1 { }; + + tensorflow-tensorboard_2 = callPackage ../development/python-modules/tensorflow-tensorboard/2 { }; + + tensorflow-bin = self.tensorflow-bin_1; + + tensorflow-bin_1 = callPackage ../development/python-modules/tensorflow/1/bin.nix { + cudaSupport = pkgs.config.cudaSupport or false; + inherit (pkgs.linuxPackages) nvidia_x11; + cudatoolkit = pkgs.cudatoolkit_10; + cudnn = pkgs.cudnn_cudatoolkit_10; + }; - tensorflow-bin = callPackage ../development/python-modules/tensorflow/bin.nix { + tensorflow-bin_2 = callPackage ../development/python-modules/tensorflow/2/bin.nix { cudaSupport = pkgs.config.cudaSupport or false; inherit (pkgs.linuxPackages) nvidia_x11; cudatoolkit = pkgs.cudatoolkit_10; cudnn = pkgs.cudnn_cudatoolkit_10; }; - tensorflow-build = callPackage ../development/python-modules/tensorflow { + tensorflow-build = self.tensorflow-build_1; + + tensorflow-build_1 = callPackage ../development/python-modules/tensorflow/1 { + cudaSupport = pkgs.config.cudaSupport or false; + inherit (pkgs.linuxPackages) nvidia_x11; + cudatoolkit = pkgs.cudatoolkit_10; + cudnn = pkgs.cudnn_cudatoolkit_10; + nccl = pkgs.nccl_cudatoolkit_10; + openssl = pkgs.openssl_1_1; + inherit (pkgs.darwin.apple_sdk.frameworks) Foundation Security; + }; + + tensorflow-build_2 = callPackage ../development/python-modules/tensorflow/2 { cudaSupport = pkgs.config.cudaSupport or false; inherit (pkgs.linuxPackages) nvidia_x11; cudatoolkit = pkgs.cudatoolkit_10; @@ -6572,7 +6600,9 @@ in { inherit (pkgs.darwin.apple_sdk.frameworks) Foundation Security; }; - tensorflow = self.tensorflow-build; + tensorflow = self.tensorflow_1; + tensorflow_1 = self.tensorflow-build_1; + tensorflow_2 = self.tensorflow-build_2; tensorflowWithoutCuda = self.tensorflow.override { cudaSupport = false; -- cgit 1.4.1