Merge commit 'bc9df0f66110039e495b6debe3a6cda4a1bb0fed'

7 files changed, 153 insertions, 36 deletions

diff --git a/nixpkgs/pkgs/build-support/docker/default.nix b/nixpkgs/pkgs/build-support/docker/default.nix
index 11945e7b6f72..57e40069003a 100644
--- a/nixpkgs/pkgs/build-support/docker/default.nix
+++ b/nixpkgs/pkgs/build-support/docker/default.nix
@@ -216,7 +216,7 @@ rec {
         find image/$extractionID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
 
         # Get the next lower directory and continue the loop.
-        lowerdir=$lowerdir''${lowerdir:+:}image/$extractionID/layer
+        lowerdir=image/$extractionID/layer''${lowerdir:+:}$lowerdir
       done
 
       mkdir work
@@ -585,9 +585,9 @@ rec {
           layerID=$(sha256sum "$layer/json" | cut -d ' ' -f 1)
           ln -s "$layer" "./image/$layerID"
 
-          manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$layerID/layer.tar\"] + .")
-          imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"$(jq -r .created ${configJson})\"}] + .")
-          imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .")
+          manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= . + [\"$layerID/layer.tar\"]")
+          imageJson=$(echo "$imageJson" | jq ".history |= . + [{\"created\": \"$(jq -r .created ${configJson})\"}]")
+          imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= . + [\"sha256:$layerChecksum\"]")
         done
         imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1)
         echo "$imageJson" > "image/$imageJsonChecksum.json"
@@ -779,23 +779,24 @@ rec {
         # Use the temp folder we've been working on to create a new image.
         mv temp image/$layerID
 
-        # Add the new layer ID to the beginning of the layer list
+        # Add the new layer ID to the end of the layer list
         (
+          cat layer-list
           # originally this used `sed -i "1i$layerID" layer-list`, but
           # would fail if layer-list was completely empty.
           echo "$layerID/layer.tar"
-          cat layer-list
         ) | ${pkgs.moreutils}/bin/sponge layer-list
 
         # Create image json and image manifest
         imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}")
         manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]")
 
-        for layerTar in $(tac ./layer-list); do
+        for layerTar in $(cat ./layer-list); do
           layerChecksum=$(sha256sum image/$layerTar | cut -d ' ' -f1)
-          imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"$(jq -r .created ${baseJson})\"}] + .")
-          imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .")
-          manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$layerTar\"] + .")
+          imageJson=$(echo "$imageJson" | jq ".history |= . + [{\"created\": \"$(jq -r .created ${baseJson})\"}]")
+          # diff_ids order is from the bottom-most to top-most layer
+          imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= . + [\"sha256:$layerChecksum\"]")
+          manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= . + [\"$layerTar\"]")
         done
 
         imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1)
diff --git a/nixpkgs/pkgs/build-support/docker/examples.nix b/nixpkgs/pkgs/build-support/docker/examples.nix
index 557a4dbf54b7..ac21be907b83 100644
--- a/nixpkgs/pkgs/build-support/docker/examples.nix
+++ b/nixpkgs/pkgs/build-support/docker/examples.nix
@@ -187,4 +187,43 @@ rec {
     runAsRoot = "touch /example-file";
     fromImage = bash;
   };
+
+  # 13. example of 3 layers images This image is used to verify the
+  # order of layers is correct.
+  # It allows to validate
+  # - the layer of parent are below
+  # - the order of parent layer is preserved at image build time
+  #   (this is why there are 3 images)
+  layersOrder = let
+    l1 = pkgs.dockerTools.buildImage {
+      name = "l1";
+      tag = "latest";
+      extraCommands = ''
+        mkdir -p tmp
+        echo layer1 > tmp/layer1
+        echo layer1 > tmp/layer2
+        echo layer1 > tmp/layer3
+      '';
+    };
+    l2 = pkgs.dockerTools.buildImage {
+      name = "l2";
+      fromImage = l1;
+      tag = "latest";
+      extraCommands = ''
+        mkdir -p tmp
+        echo layer2 > tmp/layer2
+        echo layer2 > tmp/layer3
+      '';
+    };
+  in pkgs.dockerTools.buildImage {
+    name = "l3";
+    fromImage = l2;
+    tag = "latest";
+    contents = [ pkgs.coreutils ];
+    extraCommands = ''
+      mkdir -p tmp
+      echo layer3 > tmp/layer3
+    '';
+  };
+
 }
diff --git a/nixpkgs/pkgs/build-support/fetchpatch/default.nix b/nixpkgs/pkgs/build-support/fetchpatch/default.nix
index 89d72f512f7f..2fb32b2324f2 100644
--- a/nixpkgs/pkgs/build-support/fetchpatch/default.nix
+++ b/nixpkgs/pkgs/build-support/fetchpatch/default.nix
@@ -5,6 +5,10 @@
 # stripLen acts as the -p parameter when applying a patch.
 
 { lib, fetchurl, buildPackages }:
+let
+  # 0.3.4 would change hashes: https://github.com/NixOS/nixpkgs/issues/25154
+  patchutils = buildPackages.patchutils_0_3_3;
+in
 { stripLen ? 0, extraPrefix ? null, excludes ? [], includes ? [], revert ? false, ... }@args:
 
 fetchurl ({
@@ -14,10 +18,10 @@ fetchurl ({
       echo "error: Fetched patch file '$out' is empty!" 1>&2
       exit 1
     fi
-    "${buildPackages.patchutils}/bin/lsdiff" "$out" \
+    "${patchutils}/bin/lsdiff" "$out" \
       | sort -u | sed -e 's/[*?]/\\&/g' \
       | xargs -I{} \
-        "${buildPackages.patchutils}/bin/filterdiff" \
+        "${patchutils}/bin/filterdiff" \
         --include={} \
         --strip=${toString stripLen} \
         ${lib.optionalString (extraPrefix != null) ''
@@ -32,7 +36,7 @@ fetchurl ({
       cat "$out" 1>&2
       exit 1
     fi
-    ${buildPackages.patchutils}/bin/filterdiff \
+    ${patchutils}/bin/filterdiff \
       -p1 \
       ${builtins.toString (builtins.map (x: "-x ${lib.escapeShellArg x}") excludes)} \
       ${builtins.toString (builtins.map (x: "-i ${lib.escapeShellArg x}") includes)} \
@@ -46,7 +50,7 @@ fetchurl ({
       exit 1
     fi
   '' + lib.optionalString revert ''
-    ${buildPackages.patchutils}/bin/interdiff "$out" /dev/null > "$tmpfile"
+    ${patchutils}/bin/interdiff "$out" /dev/null > "$tmpfile"
     mv "$tmpfile" "$out"
   '' + (args.postFetch or "");
   meta.broken = excludes != [] && includes != [];
diff --git a/nixpkgs/pkgs/build-support/setup-hooks/patch-shebangs.sh b/nixpkgs/pkgs/build-support/setup-hooks/patch-shebangs.sh
index d26bf735d30a..f4a865e96687 100644
--- a/nixpkgs/pkgs/build-support/setup-hooks/patch-shebangs.sh
+++ b/nixpkgs/pkgs/build-support/setup-hooks/patch-shebangs.sh
@@ -5,10 +5,32 @@
 # rewritten to /nix/store/<hash>/bin/python.  Interpreters that are
 # already in the store are left untouched.
 
-fixupOutputHooks+=('if [ -z "$dontPatchShebangs" -a -e "$prefix" ]; then patchShebangs "$prefix"; fi')
+fixupOutputHooks+=(patchShebangsAuto)
+
+# Run patch shebangs on a directory.
+# patchShebangs [--build | --host] directory
+
+# Flags:
+# --build : Lookup commands available at build-time
+# --host  : Lookup commands available at runtime
+
+# Example use cases,
+# $ patchShebangs --host /nix/store/...-hello-1.0/bin
+# $ patchShebangs --build configure
 
 patchShebangs() {
+    local pathName
+
+    if [ "$1" = "--host" ]; then
+        pathName=HOST_PATH
+        shift
+    elif [ "$1" = "--build" ]; then
+        pathName=PATH
+        shift
+    fi
+
     local dir="$1"
+
     header "patching script interpreter paths in $dir"
     local f
     local oldPath
@@ -27,6 +49,14 @@ patchShebangs() {
         oldInterpreterLine=$(head -1 "$f" | tail -c+3)
         read -r oldPath arg0 args <<< "$oldInterpreterLine"
 
+        if [ -z "$pathName" ]; then
+            if [ -n "$strictDeps" ] && [[ "$f" = "$NIX_STORE"* ]]; then
+                pathName=HOST_PATH
+            else
+                pathName=PATH
+            fi
+        fi
+
         if $(echo "$oldPath" | grep -q "/bin/env$"); then
             # Check for unsupported 'env' functionality:
             # - options: something starting with a '-'
@@ -35,14 +65,17 @@ patchShebangs() {
                 echo "$f: unsupported interpreter directive \"$oldInterpreterLine\" (set dontPatchShebangs=1 and handle shebang patching yourself)"
                 exit 1
             fi
-            newPath="$(command -v "$arg0" || true)"
+
+            newPath="$(PATH="${!pathName}" command -v "$arg0" || true)"
         else
             if [ "$oldPath" = "" ]; then
                 # If no interpreter is specified linux will use /bin/sh. Set
                 # oldpath="/bin/sh" so that we get /nix/store/.../sh.
                 oldPath="/bin/sh"
             fi
-            newPath="$(command -v "$(basename "$oldPath")" || true)"
+
+            newPath="$(PATH="${!pathName}" command -v "$(basename "$oldPath")" || true)"
+
             args="$arg0 $args"
         fi
 
@@ -55,13 +88,28 @@ patchShebangs() {
                 # escape the escape chars so that sed doesn't interpret them
                 escapedInterpreterLine=$(echo "$newInterpreterLine" | sed 's|\\|\\\\|g')
                 # Preserve times, see: https://github.com/NixOS/nixpkgs/pull/33281
-                touch -r "$f" "$f.timestamp"
+                timestamp=$(mktemp)
+                touch -r "$f" "$timestamp"
                 sed -i -e "1 s|.*|#\!$escapedInterpreterLine|" "$f"
-                touch -r "$f.timestamp" "$f"
-                rm "$f.timestamp"
+                touch -r "$timestamp" "$f"
+                rm "$timestamp"
             fi
         fi
     done < <(find "$dir" -type f -perm -0100 -print0)
 
     stopNest
 }
+
+patchShebangsAuto () {
+    if [ -z "$dontPatchShebangs" -a -e "$prefix" ]; then
+
+        # Dev output will end up being run on the build platform. An
+        # example case of this is sdl2-config. Otherwise, we can just
+        # use the runtime path (--host).
+        if [ "$output" != out ] && [ "$output" = "$outputDev" ]; then
+            patchShebangs --build "$prefix"
+        else
+            patchShebangs --host "$prefix"
+        fi
+    fi
+}
diff --git a/nixpkgs/pkgs/build-support/trivial-builders.nix b/nixpkgs/pkgs/build-support/trivial-builders.nix
index e498417adf01..f56ce7bb87d0 100644
--- a/nixpkgs/pkgs/build-support/trivial-builders.nix
+++ b/nixpkgs/pkgs/build-support/trivial-builders.nix
@@ -79,7 +79,6 @@ rec {
         (test -n "$executable" && chmod +x "$n") || true
       '';
 
-
   /*
    * Writes a text file to nix store with no optional parameters available.
    *
@@ -92,6 +91,7 @@ rec {
    *
   */
   writeText = name: text: writeTextFile {inherit name text;};
+
   /*
    * Writes a text file to nix store in a specific directory with no
    * optional parameters available. Name passed is the destination.
@@ -105,6 +105,7 @@ rec {
    *
   */
   writeTextDir = name: text: writeTextFile {inherit name text; destination = "/${name}";};
+
   /*
    * Writes a text file to /nix/store/<store path> and marks the file as executable.
    *
@@ -117,13 +118,14 @@ rec {
    *
   */
   writeScript = name: text: writeTextFile {inherit name text; executable = true;};
+
   /*
    * Writes a text file to /nix/store/<store path>/bin/<name> and
    * marks the file as executable.
    *
    * Example:
    * # Writes my-file to /nix/store/<store path>/bin/my-file and makes executable.
-   * writeScript "my-file"
+   * writeScriptBin "my-file"
    *   ''
    *   Contents of File
    *   '';
@@ -132,12 +134,38 @@ rec {
   writeScriptBin = name: text: writeTextFile {inherit name text; executable = true; destination = "/bin/${name}";};
 
   /*
-   * Writes a Shell script and check its syntax. Automatically includes interpreter
-   * above the contents passed.
+   * Similar to writeScript. Writes a Shell script and checks its syntax.
+   * Automatically includes interpreter above the contents passed.
+   *
+   * Example:
+   * # Writes my-file to /nix/store/<store path>/my-file and makes executable.
+   * writeShellScript "my-file"
+   *   ''
+   *   Contents of File
+   *   '';
+   *
+  */
+  writeShellScript = name: text:
+    writeTextFile {
+      inherit name;
+      executable = true;
+      text = ''
+        #!${runtimeShell}
+        ${text}
+        '';
+      checkPhase = ''
+        ${stdenv.shell} -n $out
+      '';
+    };
+
+  /*
+   * Similar to writeShellScript and writeScriptBin.
+   * Writes an executable Shell script to /nix/store/<store path>/bin/<name> and checks its syntax.
+   * Automatically includes interpreter above the contents passed.
    *
    * Example:
    * # Writes my-file to /nix/store/<store path>/bin/my-file and makes executable.
-   * writeScript "my-file"
+   * writeShellScriptBin "my-file"
    *   ''
    *   Contents of File
    *   '';
diff --git a/nixpkgs/pkgs/build-support/vm/default.nix b/nixpkgs/pkgs/build-support/vm/default.nix
index de6a5e3b62f9..5d04302591ee 100644
--- a/nixpkgs/pkgs/build-support/vm/default.nix
+++ b/nixpkgs/pkgs/build-support/vm/default.nix
@@ -1021,22 +1021,22 @@ rec {
     };
 
     debian9i386 = {
-      name = "debian-9.4-stretch-i386";
-      fullName = "Debian 9.4 Stretch (i386)";
+      name = "debian-9.8-stretch-i386";
+      fullName = "Debian 9.8 Stretch (i386)";
       packagesList = fetchurl {
-        url = http://snapshot.debian.org/archive/debian/20180912T154744Z/dists/stretch/main/binary-i386/Packages.xz;
-        sha256 = "0flvn8zn7vk04p10ndf3aq0mdr8k2ic01g51aq4lsllkv8lmwzyh";
+        url = http://snapshot.debian.org/archive/debian/20190503T090946Z/dists/stretch/main/binary-i386/Packages.xz;
+        sha256 = "1dr3skl35iyj85qlc33lq4whippbqf327vnbcyfqqrv6h86k68mw";
       };
       urlPrefix = mirror://debian;
       packages = commonDebianPackages;
     };
 
     debian9x86_64 = {
-      name = "debian-9.4-stretch-amd64";
-      fullName = "Debian 9.4 Stretch (amd64)";
+      name = "debian-9.8-stretch-amd64";
+      fullName = "Debian 9.8 Stretch (amd64)";
       packagesList = fetchurl {
-        url = http://snapshot.debian.org/archive/debian/20180912T154744Z/dists/stretch/main/binary-amd64/Packages.xz;
-        sha256 = "11vnn9bba2jabixvabfbw9zparl326c88xn99di7pbr5xsnl15jm";
+        url = http://snapshot.debian.org/archive/debian/20190503T090946Z/dists/stretch/main/binary-amd64/Packages.xz;
+        sha256 = "01q00nl47p12n7wx0xclx59wf3zlkzrgj3zxpshyvb91xdnw5sh6";
       };
       urlPrefix = mirror://debian;
       packages = commonDebianPackages;
diff --git a/nixpkgs/pkgs/build-support/wrapper-common/utils.bash b/nixpkgs/pkgs/build-support/wrapper-common/utils.bash
index 12b596a83e6f..4fd57162072e 100644
--- a/nixpkgs/pkgs/build-support/wrapper-common/utils.bash
+++ b/nixpkgs/pkgs/build-support/wrapper-common/utils.bash
@@ -86,9 +86,6 @@ expandResponseParams() {
                 #shellcheck disable=SC2034
                 readarray -d '' params < <("@expandResponseParams@" "$@")
                 return 0
-            else
-                echo "Response files aren't supported during bootstrapping" >&2
-                return 1
             fi
         fi
     done