Merge remote-tracking branch 'upstream/master' into release-lib-cleanup

8 files changed, 119 insertions, 27 deletions

diff --git a/pkgs/stdenv/adapters.nix b/pkgs/stdenv/adapters.nix
index 7d5277292738..671306f6e6f2 100644
--- a/pkgs/stdenv/adapters.nix
+++ b/pkgs/stdenv/adapters.nix
@@ -171,4 +171,19 @@ rec {
         NIX_CFLAGS_LINK = toString (args.NIX_CFLAGS_LINK or "") + " -fuse-ld=gold";
       });
     };
+
+
+  /* Modify a stdenv so that it builds binaries optimized specifically
+     for the machine they are built on.
+
+     WARNING: this breaks purity! */
+  impureUseNativeOptimizations = stdenv: stdenv //
+    { mkDerivation = args: stdenv.mkDerivation (args // {
+        NIX_CFLAGS_COMPILE = toString (args.NIX_CFLAGS_COMPILE or "") + " -march=native";
+        NIX_ENFORCE_NO_NATIVE = false;
+
+        preferLocalBuild = true;
+        allowSubstitutes = false;
+      });
+    };
 }
diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix
index cda643c42fe3..5fb410b64ebd 100644
--- a/pkgs/stdenv/darwin/default.nix
+++ b/pkgs/stdenv/darwin/default.nix
@@ -88,7 +88,6 @@ in rec {
         extraPackages = lib.optional (libcxx != null) libcxx;
 
         nativeTools  = false;
-        propagateDoc = false;
         nativeLibc   = false;
         inherit buildPackages coreutils gnugrep bintools;
         libc         = last.pkgs.darwin.Libsystem;
@@ -190,7 +189,6 @@ in rec {
   stage1 = prevStage: let
     persistent = self: super: with prevStage; {
       cmake = super.cmake.override {
-        majorVersion = "3.9";  # FIXME: update ApplicationServices patch
         isBootstrap = true;
         useSharedLibraries = false;
       };
@@ -199,6 +197,10 @@ in rec {
         CF = null;  # use CoreFoundation from bootstrap-tools
         configd = null;
       };
+      python2 = self.python;
+
+      ninja = super.ninja.override { buildDocs = false; };
+      darwin = super.darwin // { cctools = super.darwin.cctools.override { llvm = null; }; };
     };
   in with prevStage; stageFun 1 prevStage {
     extraPreHook = "export NIX_CFLAGS_COMPILE+=\" -F${bootstrapTools}/Library/Frameworks\"";
@@ -218,11 +220,12 @@ in rec {
         zlib patchutils m4 scons flex perl bison unifdef unzip openssl python
         libxml2 gettext sharutils gmp libarchive ncurses pkg-config libedit groff
         openssh sqlite sed serf openldap db cyrus-sasl expat apr-util subversion xz
-        findfreetype libssh curl cmake autoconf automake libtool ed cpio coreutils;
+        findfreetype libssh curl cmake autoconf automake libtool ed cpio coreutils
+        libssh2 nghttp2 libkrb5 python2 ninja;
 
       darwin = super.darwin // {
         inherit (darwin)
-          dyld Libsystem xnu configd ICU libdispatch libclosure launchd;
+          dyld Libsystem xnu configd ICU libdispatch libclosure launchd CF;
       };
     };
   in with prevStage; stageFun 2 prevStage {
@@ -236,7 +239,10 @@ in rec {
 
     allowedRequisites =
       [ bootstrapTools ] ++
-      (with pkgs; [ xz.bin xz.out libcxx libcxxabi ]) ++
+      (with pkgs; [
+        xz.bin xz.out libcxx libcxxabi zlib libxml2.out curl.out openssl.out libssh2.out
+        nghttp2.lib libkrb5
+      ]) ++
       (with pkgs.darwin; [ dyld Libsystem CF ICU locale ]);
 
     overrides = persistent;
@@ -248,9 +254,10 @@ in rec {
         patchutils m4 scons flex perl bison unifdef unzip openssl python
         gettext sharutils libarchive pkg-config groff bash subversion
         openssh sqlite sed serf openldap db cyrus-sasl expat apr-util
-        findfreetype libssh curl cmake autoconf automake libtool cpio;
+        findfreetype libssh curl cmake autoconf automake libtool cpio
+        libssh2 nghttp2 libkrb5 python2 ninja;
 
-      # Avoid pulling in a full python and it's extra dependencies for the llvm/clang builds.
+      # Avoid pulling in a full python and its extra dependencies for the llvm/clang builds.
       libxml2 = super.libxml2.override { pythonSupport = false; };
 
       llvmPackages_5 = super.llvmPackages_5 // (let
@@ -282,7 +289,10 @@ in rec {
 
     allowedRequisites =
       [ bootstrapTools ] ++
-      (with pkgs; [ xz.bin xz.out bash libcxx libcxxabi ]) ++
+      (with pkgs; [
+        xz.bin xz.out bash libcxx libcxxabi zlib libxml2.out curl.out openssl.out libssh2.out
+        nghttp2.lib libkrb5
+      ]) ++
       (with pkgs.darwin; [ dyld ICU Libsystem locale ]);
 
     overrides = persistent;
@@ -293,7 +303,7 @@ in rec {
       inherit
         gnumake gzip gnused bzip2 gawk ed xz patch bash
         ncurses libffi zlib gmp pcre gnugrep
-        coreutils findutils diffutils patchutils;
+        coreutils findutils diffutils patchutils ninja;
 
       # Hack to make sure we don't link ncurses in bootstrap tools. The proper
       # solution is to avoid passing -L/nix-store/...-bootstrap-tools/lib,
@@ -313,8 +323,14 @@ in rec {
         });
       in { inherit tools libraries; } // tools // libraries);
 
-      darwin = super.darwin // {
+      darwin = super.darwin // rec {
         inherit (darwin) dyld Libsystem libiconv locale;
+
+        libxml2-nopython = super.libxml2.override { pythonSupport = false; };
+        CF = super.darwin.CF.override {
+          libxml2 = libxml2-nopython;
+          python = prevStage.python;
+        };
       };
     };
   in with prevStage; stageFun 4 prevStage {
@@ -346,6 +362,17 @@ in rec {
         });
       in { inherit tools libraries; } // tools // libraries);
 
+      # N.B: the important thing here is to ensure that python == python2
+      # == python27 or you get weird issues with inconsistent package sets.
+      # In a particularly subtle bug, I overrode python2 instead of python27
+      # here, and it caused gnome-doc-utils to complain about:
+      # "PyThreadState_Get: no current thread". This is because Python gets
+      # really unhappy if you have Python A which loads a native python lib
+      # which was linked against Python B, which in our case was happening
+      # because we didn't override python "deeply enough". Anyway, this works
+      # and I'm just leaving this blurb here so people realize why it matters
+      python27 = super.python27.override { CF = prevStage.darwin.CF; };
+
       darwin = super.darwin // {
         inherit (darwin) dyld ICU Libsystem libiconv;
       } // lib.optionalAttrs (super.stdenv.targetPlatform == localSystem) {
@@ -399,9 +426,10 @@ in rec {
       gzip ncurses.out ncurses.dev ncurses.man gnused bash gawk
       gnugrep llvmPackages.clang-unwrapped llvmPackages.clang-unwrapped.lib patch pcre.out gettext
       binutils.bintools darwin.binutils darwin.binutils.bintools
+      curl.out openssl.out libssh2.out nghttp2.lib libkrb5
       cc.expand-response-params
     ]) ++ (with pkgs.darwin; [
-      dyld Libsystem CF cctools ICU libiconv locale
+      dyld Libsystem CF cctools ICU libiconv locale libxml2-nopython.out
     ]);
 
     overrides = lib.composeExtensions persistent (self: super: {
diff --git a/pkgs/stdenv/darwin/make-bootstrap-tools.nix b/pkgs/stdenv/darwin/make-bootstrap-tools.nix
index 66c5f419f2f6..eee3b1ce0759 100644
--- a/pkgs/stdenv/darwin/make-bootstrap-tools.nix
+++ b/pkgs/stdenv/darwin/make-bootstrap-tools.nix
@@ -3,7 +3,7 @@
 with import pkgspath { inherit system; };
 
 let
-  llvmPackages = llvmPackages_4;
+  llvmPackages = llvmPackages_5;
 in rec {
   coreutils_ = coreutils.override (args: {
     # We want coreutils without ACL support.
@@ -12,6 +12,10 @@ in rec {
     singleBinary = false;
   });
 
+  # We want a version of cctools without LLVM, because the LTO support ends up making
+  # the bootstrap tools huge and isn't really necessary for bootstrap
+  cctools_ = darwin.cctools.override { llvm = null; };
+
   # Avoid debugging larger changes for now.
   bzip2_ = bzip2.override (args: { linkStatic = true; });
 
@@ -73,6 +77,7 @@ in rec {
       cp -d ${gettext}/lib/libintl*.dylib $out/lib
       chmod +x $out/lib/libintl*.dylib
       cp -d ${ncurses.out}/lib/libncurses*.dylib $out/lib
+      cp -d ${libxml2.out}/lib/libxml2*.dylib $out/lib
 
       # Copy what we need of clang
       cp -d ${llvmPackages.clang-unwrapped}/bin/clang $out/bin
@@ -94,7 +99,7 @@ in rec {
 
       # Copy binutils.
       for i in as ld ar ranlib nm strip otool install_name_tool dsymutil lipo; do
-        cp ${darwin.cctools}/bin/$i $out/bin
+        cp ${cctools_}/bin/$i $out/bin
       done
 
       cp -rd ${pkgs.darwin.CF}/Library $out
@@ -104,9 +109,9 @@ in rec {
       nuke-refs $out/bin/*
 
       rpathify() {
-        local libs=$(${darwin.cctools}/bin/otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*") || true
+        local libs=$(${cctools_}/bin/otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*") || true
         for lib in $libs; do
-          ${darwin.cctools}/bin/install_name_tool -change $lib "@rpath/$(basename $lib)" "$1"
+          ${cctools_}/bin/install_name_tool -change $lib "@rpath/$(basename $lib)" "$1"
         done
       }
 
diff --git a/pkgs/stdenv/generic/check-meta.nix b/pkgs/stdenv/generic/check-meta.nix
index a5c8ca705231..26cd9f8beb96 100644
--- a/pkgs/stdenv/generic/check-meta.nix
+++ b/pkgs/stdenv/generic/check-meta.nix
@@ -42,8 +42,7 @@ let
   allowUnsupportedSystem = config.allowUnsupportedSystem or false
     || builtins.getEnv "NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM" == "1";
 
-  isUnfree = licenses: lib.lists.any (l:
-    !l.free or true || l == "unfree" || l == "unfree-redistributable") licenses;
+  isUnfree = licenses: lib.lists.any (l: !l.free or true) licenses;
 
   # Alow granular checks to allow only some unfree packages
   # Example:
@@ -56,7 +55,7 @@ let
 
   # Check whether unfree packages are allowed and if not, whether the
   # package has an unfree license and is not explicitely allowed by the
-  # `allowUNfreePredicate` function.
+  # `allowUnfreePredicate` function.
   hasDeniedUnfreeLicense = attrs:
     !allowUnfree &&
     hasLicense attrs &&
@@ -81,6 +80,7 @@ let
     unsupported = remediate_whitelist "UnsupportedSystem";
     blacklisted = x: "";
     insecure = remediate_insecure;
+    broken-outputs = remediateOutputsToInstall;
     unknown-meta = x: "";
   };
   remediate_whitelist = allow_attr: attrs:
@@ -125,6 +125,20 @@ let
 
       '';
 
+  remediateOutputsToInstall = attrs: let
+      expectedOutputs = attrs.meta.outputsToInstall or [];
+      actualOutputs = attrs.outputs or [ "out" ];
+      missingOutputs = builtins.filter (output: ! builtins.elem output actualOutputs) expectedOutputs;
+    in ''
+      The package ${attrs.name} has set meta.outputsToInstall to: ${builtins.concatStringsSep ", " expectedOutputs}
+
+      however ${attrs.name} only has the outputs: ${builtins.concatStringsSep ", " actualOutputs}
+
+      and is missing the following ouputs:
+
+      ${lib.concatStrings (builtins.map (output: "  - ${output}\n") missingOutputs)}
+    '';
+
   handleEvalIssue = attrs: { reason , errormsg ? "" }:
     let
       msg = ''
@@ -185,6 +199,14 @@ let
     in  anyMatch (attrs.meta.platforms or lib.platforms.all) &&
       ! anyMatch (attrs.meta.badPlatforms or []);
 
+  checkOutputsToInstall = attrs: let
+      expectedOutputs = attrs.meta.outputsToInstall or [];
+      actualOutputs = attrs.outputs or [ "out" ];
+      missingOutputs = builtins.filter (output: ! builtins.elem output actualOutputs) expectedOutputs;
+    in if shouldCheckMeta
+       then builtins.length missingOutputs > 0
+       else false;
+
   # Check if a derivation is valid, that is whether it passes checks for
   # e.g brokenness or license.
   #
@@ -202,6 +224,8 @@ let
       { valid = false; reason = "unsupported"; errormsg = "is not supported on ‘${hostPlatform.config}’"; }
     else if !(hasAllowedInsecure attrs) then
       { valid = false; reason = "insecure"; errormsg = "is marked as insecure"; }
+    else if checkOutputsToInstall attrs then
+      { valid = false; reason = "broken-outputs"; errormsg = "has invalid meta.outputsToInstall"; }
     else let res = checkMeta (attrs.meta or {}); in if res != [] then
       { valid = false; reason = "unknown-meta"; errormsg = "has an invalid meta attrset:${lib.concatMapStrings (x: "\n\t - " + x) res}"; }
     else { valid = true; };
diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix
index c663c3743ed6..e06faed30a1e 100644
--- a/pkgs/stdenv/generic/make-derivation.nix
+++ b/pkgs/stdenv/generic/make-derivation.nix
@@ -41,6 +41,7 @@ rec {
 
     # Configure Phase
     , configureFlags ? []
+    , cmakeFlags ? []
     , # Target is not included by default because most programs don't care.
       # Including it then would cause needless mass rebuilds.
       #
@@ -225,6 +226,19 @@ rec {
           inherit doCheck doInstallCheck;
 
           inherit outputs;
+        } // lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform) {
+          cmakeFlags =
+            (/**/ if lib.isString cmakeFlags then [cmakeFlags]
+             else if cmakeFlags == null      then []
+             else                                     cmakeFlags)
+          ++ lib.optional (stdenv.hostPlatform.uname.system != null) "-DCMAKE_SYSTEM_NAME=${stdenv.hostPlatform.uname.system}"
+          ++ lib.optional (stdenv.hostPlatform.uname.processor != null) "-DCMAKE_SYSTEM_PROCESSOR=${stdenv.hostPlatform.uname.processor}"
+          ++ lib.optional (stdenv.hostPlatform.uname.release != null) "-DCMAKE_SYSTEM_VERSION=${stdenv.hostPlatform.release}"
+          ++ lib.optional (stdenv.buildPlatform.uname.system != null) "-DCMAKE_HOST_SYSTEM_NAME=${stdenv.buildPlatform.uname.system}"
+          ++ lib.optional (stdenv.buildPlatform.uname.processor != null) "-DCMAKE_HOST_SYSTEM_PROCESSOR=${stdenv.buildPlatform.uname.processor}"
+          ++ lib.optional (stdenv.buildPlatform.uname.release != null) "-DCMAKE_HOST_SYSTEM_VERSION=${stdenv.buildPlatform.uname.release}";
+        } // lib.optionalAttrs (attrs.enableParallelBuilding or false) {
+          enableParallelChecking = attrs.enableParallelChecking or true;
         } // lib.optionalAttrs (hardeningDisable != [] || hardeningEnable != []) {
           NIX_HARDENING_ENABLE = enabledHardeningOptions;
         } // lib.optionalAttrs (stdenv.buildPlatform.isDarwin) {
diff --git a/pkgs/stdenv/generic/setup.sh b/pkgs/stdenv/generic/setup.sh
index 141e94c5ed46..8af369b1d17d 100644
--- a/pkgs/stdenv/generic/setup.sh
+++ b/pkgs/stdenv/generic/setup.sh
@@ -211,7 +211,7 @@ isELF() {
     exec {fd}< "$fn"
     read -r -n 4 -u "$fd" magic
     exec {fd}<&-
-    if [[ "$magic" =~ ELF ]]; then return 0; else return 1; fi
+    if [ "$magic" = $'\177ELF' ]; then return 0; else return 1; fi
 }
 
 # Return success if the specified file is a script (i.e. starts with
@@ -257,9 +257,17 @@ shopt -s nullglob
 
 # Set up the initial path.
 PATH=
+HOST_PATH=
 for i in $initialPath; do
     if [ "$i" = / ]; then i=; fi
     addToSearchPath PATH "$i/bin"
+
+    # For backward compatibility, we add initial path to HOST_PATH so
+    # it can be used in auto patch-shebangs. Unfortunately this will
+    # not work with cross compilation.
+    if [ -z "${strictDeps-}" ]; then
+        addToSearchPath HOST_PATH "$i/bin"
+    fi
 done
 
 if (( "${NIX_DEBUG:-0}" >= 1 )); then
@@ -272,7 +280,6 @@ if [ -z "${SHELL:-}" ]; then echo "SHELL not set"; exit 1; fi
 BASH="$SHELL"
 export CONFIG_SHELL="$SHELL"
 
-
 # Dummy implementation of the paxmark function. On Linux, this is
 # overwritten by paxctl's setup hook.
 paxmark() { true; }
@@ -1044,7 +1051,7 @@ checkPhase() {
         # Old bash empty array hack
         # shellcheck disable=SC2086
         local flagsArray=(
-            ${enableParallelBuilding:+-j${NIX_BUILD_CORES} -l${NIX_BUILD_CORES}}
+            ${enableParallelChecking:+-j${NIX_BUILD_CORES} -l${NIX_BUILD_CORES}}
             $makeFlags ${makeFlagsArray+"${makeFlagsArray[@]}"}
             ${checkFlags:-VERBOSE=y} ${checkFlagsArray+"${checkFlagsArray[@]}"}
             ${checkTarget}
@@ -1176,7 +1183,7 @@ installCheckPhase() {
         # Old bash empty array hack
         # shellcheck disable=SC2086
         local flagsArray=(
-            ${enableParallelBuilding:+-j${NIX_BUILD_CORES} -l${NIX_BUILD_CORES}}
+            ${enableParallelChecking:+-j${NIX_BUILD_CORES} -l${NIX_BUILD_CORES}}
             $makeFlags ${makeFlagsArray+"${makeFlagsArray[@]}"}
             $installCheckFlags ${installCheckFlagsArray+"${installCheckFlagsArray[@]}"}
             ${installCheckTarget:-installcheck}
diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix
index 884730cfe904..978beea692c6 100644
--- a/pkgs/stdenv/linux/default.nix
+++ b/pkgs/stdenv/linux/default.nix
@@ -92,7 +92,6 @@ let
              else lib.makeOverridable (import ../../build-support/cc-wrapper) {
           name = "${name}-gcc-wrapper";
           nativeTools = false;
-          propagateDoc = false;
           nativeLibc = false;
           buildPackages = lib.optionalAttrs (prevStage ? stdenv) {
             inherit (prevStage) stdenv;
@@ -346,8 +345,8 @@ in
       # Mainly avoid reference to bootstrap tools
       allowedRequisites = with prevStage; with lib;
         # Simple executable tools
-        concatMap (p: [ (getBin p) (getLib p) ])
-          [ gzip bzip2 xz bash binutils.bintools coreutils diffutils findutils
+        concatMap (p: [ (getBin p) (getLib p) ]) [
+            gzip bzip2 xz bash binutils.bintools coreutils diffutils findutils
             gawk gnumake gnused gnutar gnugrep gnupatch patchelf ed paxctl
           ]
         # Library dependencies
diff --git a/pkgs/stdenv/linux/make-bootstrap-tools.nix b/pkgs/stdenv/linux/make-bootstrap-tools.nix
index 58104e6ce260..f4184b92bd5f 100644
--- a/pkgs/stdenv/linux/make-bootstrap-tools.nix
+++ b/pkgs/stdenv/linux/make-bootstrap-tools.nix
@@ -112,8 +112,8 @@ in with pkgs; rec {
         cp -d ${gcc.cc.out}/bin/gcc $out/bin
         cp -d ${gcc.cc.out}/bin/cpp $out/bin
         cp -d ${gcc.cc.out}/bin/g++ $out/bin
-        cp -d ${gcc.cc.lib}/lib*/libgcc_s.so* $out/lib
-        cp -d ${gcc.cc.lib}/lib*/libstdc++.so* $out/lib
+        cp -d ${gcc.cc.lib}/lib/libgcc_s.so* $out/lib
+        cp -d ${gcc.cc.lib}/lib/libstdc++.so* $out/lib
         cp -rd ${gcc.cc.out}/lib/gcc $out/lib
         chmod -R u+w $out/lib
         rm -f $out/lib/gcc/*/*/include*/linux