From 736848723e5aefa5d24396c58dc6de603399efde Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Tue, 3 Apr 2018 10:26:03 +0200 Subject: dockerTools.pullImage: Skopeo pulls images by digest Skopeo is used to pull images from a Docker registry (instead of a Docker deamon in a VM). An image reference is specified with its name and its digest which is an immutable image identifier (unlike image name and tag). Skopeo can be used to get the digest of an image, for instance: $ skopeo inspect docker://docker.io/nixos/nix:1.11 | jq -r '.Digest' --- pkgs/build-support/docker/default.nix | 25 +++++++++++++++++++++-- pkgs/build-support/docker/examples.nix | 6 +++--- pkgs/build-support/docker/pull.nix | 32 ------------------------------ pkgs/build-support/docker/pull.sh | 36 ---------------------------------- 4 files changed, 26 insertions(+), 73 deletions(-) delete mode 100644 pkgs/build-support/docker/pull.nix delete mode 100644 pkgs/build-support/docker/pull.sh (limited to 'pkgs/build-support') diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 75e279afdc37..584beb3d89b8 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -32,7 +32,28 @@ rec { inherit pkgs buildImage pullImage shadowSetup buildImageWithNixDb; }; - pullImage = callPackage ./pull.nix {}; + pullImage = + let + fixName = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name; + in { + imageName, + # To find the digest of an image, you can use skopeo: + # skopeo inspect docker://docker.io/nixos/nix:1.11 | jq -r '.Digest' + # sha256:20d9485b25ecfd89204e843a962c1bd70e9cc6858d65d7f5fadc340246e2116b + imageDigest, + sha256, + # This used to set a tag to the pulled image + finalImageTag ? "latest", + name ? (fixName "docker-image-${imageName}-${finalImageTag}.tar") }: + runCommand name { + impureEnvVars=pkgs.stdenv.lib.fetchers.proxyImpureEnvVars; + outputHashMode="flat"; + outputHashAlgo="sha256"; + outputHash=sha256; + } + '' + ${pkgs.skopeo}/bin/skopeo copy docker://${imageName}@${imageDigest} docker-archive://$out:${imageName}:${finalImageTag} + ''; # We need to sum layer.tar, not a directory, hence tarsum instead of nix-hash. # And we cannot untar it, because then we cannot preserve permissions ecc. @@ -560,7 +581,7 @@ rec { chmod -R a-w image echo "Cooking the image..." - tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'./':: -c . | pigz -nT > $out + tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nT > $out echo "Finished." ''; diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index 315440349b6b..eb5b9fe36e41 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -85,9 +85,9 @@ rec { # 4. example of pulling an image. could be used as a base for other images nixFromDockerHub = pullImage { imageName = "nixos/nix"; - imageTag = "1.11"; - # this hash will need change if the tag is updated at docker hub - sha256 = "0nncn9pn5miygan51w34c2p9qssi96jgsaqv44dxxdprc8pg0g83"; + imageDigest = "sha256:20d9485b25ecfd89204e843a962c1bd70e9cc6858d65d7f5fadc340246e2116b"; + sha256 = "0mqjy3zq2v6rrhizgb9nvhczl87lcfphq9601wcprdika2jz7qh8"; + finalImageTag = "1.11"; }; # 5. example of multiple contents, emacs and vi happily coexisting diff --git a/pkgs/build-support/docker/pull.nix b/pkgs/build-support/docker/pull.nix deleted file mode 100644 index 5611c7785862..000000000000 --- a/pkgs/build-support/docker/pull.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ stdenv, lib, docker, vmTools, utillinux, curl, kmod, dhcp, cacert, e2fsprogs }: -let - nameReplace = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name; -in -# For simplicity we only support sha256. -{ imageName, imageTag ? "latest", imageId ? "${imageName}:${imageTag}" -, sha256, name ? (nameReplace "docker-image-${imageName}-${imageTag}.tar") }: -let - pullImage = vmTools.runInLinuxVM ( - stdenv.mkDerivation { - inherit name imageId; - - certs = "${cacert}/etc/ssl/certs/ca-bundle.crt"; - - builder = ./pull.sh; - - nativeBuildInputs = [ curl utillinux docker kmod dhcp cacert e2fsprogs ]; - - outputHashAlgo = "sha256"; - outputHash = sha256; - - impureEnvVars = lib.fetchers.proxyImpureEnvVars; - - preVM = vmTools.createEmptyImage { - size = 2048; - fullName = "${name}-disk"; - }; - - QEMU_OPTS = "-netdev user,id=net0 -device virtio-net-pci,netdev=net0"; - }); -in - pullImage diff --git a/pkgs/build-support/docker/pull.sh b/pkgs/build-support/docker/pull.sh deleted file mode 100644 index 0b1e9f310ee9..000000000000 --- a/pkgs/build-support/docker/pull.sh +++ /dev/null @@ -1,36 +0,0 @@ -source $stdenv/setup - -mkdir -p /var/lib/docker -mkfs.ext4 /dev/vda -mount -t ext4 /dev/vda /var/lib/docker - -modprobe virtio_net -dhclient eth0 - -mkdir -p /etc/ssl/certs/ -cp "$certs" "/etc/ssl/certs/" - -# from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount -mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup -cd /sys/fs/cgroup -for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do - mkdir -p $sys - if ! mountpoint -q $sys; then - if ! mount -n -t cgroup -o $sys cgroup $sys; then - rmdir $sys || true - fi - fi -done - -# run docker daemon -dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock & - -until docker ps 2>/dev/null; do - printf '.' - sleep 1 -done - -rm -r $out - -docker pull ${imageId} -docker save ${imageId} > $out -- cgit 1.4.1 From f7abcb0752c044c59766d53dc48a28d213214fb7 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Thu, 3 May 2018 11:08:25 +0100 Subject: fetchs3: allow to name the derivation output (#39823) * fetchs3: add configurable name Change the default from "foo" to the basename of the s3 URL and make it configurable. * fetchs3: fix error on missing credentials.session_token The session token should default to null instead of failing * fetchs3: make use of the region argument Set it to null if you don't want to use it * fetchs3: prefer local build Fetcher-types spend more time on network than CPU --- pkgs/build-support/fetchs3/default.nix | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'pkgs/build-support') diff --git a/pkgs/build-support/fetchs3/default.nix b/pkgs/build-support/fetchs3/default.nix index e6b7a3418c0c..14dac9997d94 100644 --- a/pkgs/build-support/fetchs3/default.nix +++ b/pkgs/build-support/fetchs3/default.nix @@ -1,6 +1,7 @@ { stdenvNoCC, runCommand, awscli }: { s3url +, name ? builtins.baseNameOf s3url , sha256 , region ? "us-east-1" , credentials ? null # Default to looking at local EC2 metadata service @@ -10,16 +11,23 @@ }: let - credentialAttrs = stdenvNoCC.lib.optionalAttrs (credentials != null) { - AWS_ACCESS_KEY_ID = credentials.access_key_id; - AWS_SECRET_ACCESS_KEY = credentials.secret_access_key; - AWS_SESSION_TOKEN = credentials.session_token ? null; + mkCredentials = { access_key_id, secret_access_key, session_token ? null }: { + AWS_ACCESS_KEY_ID = access_key_id; + AWS_SECRET_ACCESS_KEY = secret_access_key; + AWS_SESSION_TOKEN = session_token; }; -in runCommand "foo" ({ + + credentialAttrs = stdenvNoCC.lib.optionalAttrs (credentials != null) (mkCredentials credentials); +in runCommand name ({ nativeBuildInputs = [ awscli ]; + outputHashAlgo = "sha256"; outputHash = sha256; outputHashMode = if recursiveHash then "recursive" else "flat"; + + preferLocalBuild = true; + + AWS_DEFAULT_REGION = region; } // credentialAttrs) (if postFetch != null then '' downloadedFile="$(mktemp)" aws s3 cp ${s3url} $downloadedFile -- cgit 1.4.1