Merge remote-tracking branch 'upstream/master' into staging

5 files changed, 39 insertions, 78 deletions

diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
index 75e279afdc37..584beb3d89b8 100644
--- a/pkgs/build-support/docker/default.nix
+++ b/pkgs/build-support/docker/default.nix
@@ -32,7 +32,28 @@ rec {
     inherit pkgs buildImage pullImage shadowSetup buildImageWithNixDb;
   };
 
-  pullImage = callPackage ./pull.nix {};
+  pullImage =
+    let
+      fixName = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name;
+    in {
+      imageName,
+      # To find the digest of an image, you can use skopeo:
+      # skopeo inspect docker://docker.io/nixos/nix:1.11 | jq -r '.Digest'
+      # sha256:20d9485b25ecfd89204e843a962c1bd70e9cc6858d65d7f5fadc340246e2116b
+      imageDigest,
+      sha256,
+      # This used to set a tag to the pulled image
+      finalImageTag ? "latest",
+      name ? (fixName "docker-image-${imageName}-${finalImageTag}.tar") }:
+      runCommand name {
+        impureEnvVars=pkgs.stdenv.lib.fetchers.proxyImpureEnvVars;
+        outputHashMode="flat";
+        outputHashAlgo="sha256";
+        outputHash=sha256;
+      }
+      ''
+        ${pkgs.skopeo}/bin/skopeo copy docker://${imageName}@${imageDigest} docker-archive://$out:${imageName}:${finalImageTag}
+      '';
 
   # We need to sum layer.tar, not a directory, hence tarsum instead of nix-hash.
   # And we cannot untar it, because then we cannot preserve permissions ecc.
@@ -560,7 +581,7 @@ rec {
         chmod -R a-w image
 
         echo "Cooking the image..."
-        tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'./':: -c . | pigz -nT > $out
+        tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nT > $out
 
         echo "Finished."
       '';
diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix
index 315440349b6b..eb5b9fe36e41 100644
--- a/pkgs/build-support/docker/examples.nix
+++ b/pkgs/build-support/docker/examples.nix
@@ -85,9 +85,9 @@ rec {
   # 4. example of pulling an image. could be used as a base for other images
   nixFromDockerHub = pullImage {
     imageName = "nixos/nix";
-    imageTag = "1.11";
-    # this hash will need change if the tag is updated at docker hub
-    sha256 = "0nncn9pn5miygan51w34c2p9qssi96jgsaqv44dxxdprc8pg0g83";
+    imageDigest = "sha256:20d9485b25ecfd89204e843a962c1bd70e9cc6858d65d7f5fadc340246e2116b";
+    sha256 = "0mqjy3zq2v6rrhizgb9nvhczl87lcfphq9601wcprdika2jz7qh8";
+    finalImageTag = "1.11";
   };
 
   # 5. example of multiple contents, emacs and vi happily coexisting
diff --git a/pkgs/build-support/docker/pull.nix b/pkgs/build-support/docker/pull.nix
deleted file mode 100644
index 5611c7785862..000000000000
--- a/pkgs/build-support/docker/pull.nix
+++ /dev/null
@@ -1,32 +0,0 @@
-{ stdenv, lib, docker, vmTools, utillinux, curl, kmod, dhcp, cacert, e2fsprogs }:
-let
-  nameReplace = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name;
-in
-# For simplicity we only support sha256.
-{ imageName, imageTag ? "latest", imageId ? "${imageName}:${imageTag}"
-, sha256, name ? (nameReplace "docker-image-${imageName}-${imageTag}.tar") }:
-let
-  pullImage = vmTools.runInLinuxVM (
-    stdenv.mkDerivation {
-      inherit name imageId;
-
-      certs = "${cacert}/etc/ssl/certs/ca-bundle.crt";
-
-      builder = ./pull.sh;
-
-      nativeBuildInputs = [ curl utillinux docker kmod dhcp cacert e2fsprogs ];
-
-      outputHashAlgo = "sha256";
-      outputHash = sha256;
-
-      impureEnvVars = lib.fetchers.proxyImpureEnvVars;
-
-      preVM = vmTools.createEmptyImage {
-        size = 2048;
-        fullName = "${name}-disk";
-      };
-
-      QEMU_OPTS = "-netdev user,id=net0 -device virtio-net-pci,netdev=net0";
-    });
-in
-  pullImage
diff --git a/pkgs/build-support/docker/pull.sh b/pkgs/build-support/docker/pull.sh
deleted file mode 100644
index 0b1e9f310ee9..000000000000
--- a/pkgs/build-support/docker/pull.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-source $stdenv/setup
-
-mkdir -p /var/lib/docker
-mkfs.ext4 /dev/vda
-mount -t ext4 /dev/vda /var/lib/docker
-
-modprobe virtio_net
-dhclient eth0
-
-mkdir -p /etc/ssl/certs/
-cp "$certs" "/etc/ssl/certs/"
-
-# from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount
-mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
-cd /sys/fs/cgroup
-for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
-  mkdir -p $sys
-  if ! mountpoint -q $sys; then
-    if ! mount -n -t cgroup -o $sys cgroup $sys; then
-      rmdir $sys || true
-    fi
-  fi
-done
-
-# run docker daemon
-dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock &
-
-until docker ps 2>/dev/null; do
-  printf '.'
-  sleep 1
-done
-
-rm -r $out
-
-docker pull ${imageId}
-docker save ${imageId} > $out
diff --git a/pkgs/build-support/fetchs3/default.nix b/pkgs/build-support/fetchs3/default.nix
index e6b7a3418c0c..14dac9997d94 100644
--- a/pkgs/build-support/fetchs3/default.nix
+++ b/pkgs/build-support/fetchs3/default.nix
@@ -1,6 +1,7 @@
 { stdenvNoCC, runCommand, awscli }:
 
 { s3url
+, name ? builtins.baseNameOf s3url
 , sha256
 , region ? "us-east-1"
 , credentials ? null # Default to looking at local EC2 metadata service
@@ -10,16 +11,23 @@
 }:
 
 let
-  credentialAttrs = stdenvNoCC.lib.optionalAttrs (credentials != null) {
-    AWS_ACCESS_KEY_ID = credentials.access_key_id;
-    AWS_SECRET_ACCESS_KEY = credentials.secret_access_key;
-    AWS_SESSION_TOKEN = credentials.session_token ? null;
+  mkCredentials = { access_key_id, secret_access_key, session_token ? null }: {
+    AWS_ACCESS_KEY_ID = access_key_id;
+    AWS_SECRET_ACCESS_KEY = secret_access_key;
+    AWS_SESSION_TOKEN = session_token;
   };
-in runCommand "foo" ({
+
+  credentialAttrs = stdenvNoCC.lib.optionalAttrs (credentials != null) (mkCredentials credentials);
+in runCommand name ({
   nativeBuildInputs = [ awscli ];
+
   outputHashAlgo = "sha256";
   outputHash = sha256;
   outputHashMode = if recursiveHash then "recursive" else "flat";
+
+  preferLocalBuild = true;
+
+  AWS_DEFAULT_REGION = region;
 } // credentialAttrs) (if postFetch != null then ''
   downloadedFile="$(mktemp)"
   aws s3 cp ${s3url} $downloadedFile