From 8d3cb0af9e24b867df45fc7f35eddbf3e4449404 Mon Sep 17 00:00:00 2001 From: Matej Cotman Date: Fri, 26 May 2017 18:06:24 +0200 Subject: docker tools: fix pull image function --- pkgs/build-support/docker/examples.nix | 16 ++---- pkgs/build-support/docker/pull.nix | 57 ++++++++---------- pkgs/build-support/docker/pull.sh | 102 +++++++++------------------------ 3 files changed, 56 insertions(+), 119 deletions(-) (limited to 'pkgs/build-support') diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index 05b4a9b4f2d2..49cbb7a98175 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -83,16 +83,12 @@ rec { }; # 4. example of pulling an image. could be used as a base for other images - # - # ***** Currently broken, getting 404s. Perhaps the docker API has changed? - # - # - # debian = pullImage { - # imageName = "debian"; - # imageTag = "jessie"; - # # this hash will need change if the tag is updated at docker hub - # sha256 = "18kd495lc2k35h03bpcbdjnix17nlqbwf6nmq3sb161blf0dk14q"; - # }; + nix = pullImage { + imageName = "nixos/nix"; + imageTag = "1.11"; + # this hash will need change if the tag is updated at docker hub + sha256 = "1gk4bq05vl3rj3mh4mlbl4iicgndmimlv8jvkhdk4hrv0r44bwr3"; + }; # 5. example of multiple contents, emacs and vi happily coexisting editors = buildImage { diff --git a/pkgs/build-support/docker/pull.nix b/pkgs/build-support/docker/pull.nix index 0e1b147f6e18..5ccd0a41c5e4 100644 --- a/pkgs/build-support/docker/pull.nix +++ b/pkgs/build-support/docker/pull.nix @@ -1,41 +1,32 @@ -{ stdenv, lib, curl, jshon, python, runCommand }: - -# Inspired and simplified version of fetchurl. +{ stdenv, lib, docker, vmTools, utillinux, curl, kmod, dhcp, cacert, e2fsprogs }: +let + nameReplace = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name; +in # For simplicity we only support sha256. +{ imageName, imageTag ? "latest", imageId ? "${imageName}:${imageTag}" +, sha256, name ? (nameReplace "docker-image-${imageName}-${imageTag}.tar") }: +let + pullImage = vmTools.runInLinuxVM ( + stdenv.mkDerivation { + inherit name imageId; -# Currently only registry v1 is supported, compatible with Docker Hub. - -{ imageName, imageTag ? "latest", imageId ? null -, sha256, name ? "${imageName}-${imageTag}" -, indexUrl ? "https://index.docker.io" -, registryVersion ? "v1" -, curlOpts ? "" }: - -assert registryVersion == "v1"; - -let layer = stdenv.mkDerivation { - inherit name imageName imageTag imageId - indexUrl registryVersion curlOpts; + certs = "${cacert}/etc/ssl/certs/ca-bundle.crt"; - builder = ./pull.sh; - detjson = ./detjson.py; + builder = ./pull.sh; - buildInputs = [ curl jshon python ]; + buildInputs = [ curl utillinux docker kmod dhcp cacert e2fsprogs ]; - outputHashAlgo = "sha256"; - outputHash = sha256; - outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = sha256; - impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ - # This variable allows the user to pass additional options to curl - "NIX_CURL_FLAGS" - ]; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; - # Doing the download on a remote machine just duplicates network - # traffic, so don't do that. - preferLocalBuild = true; -}; + preVM = vmTools.createEmptyImage { + size = 2048; + fullName = "${name}-disk"; + }; -in runCommand "${name}.tar.gz" {} '' - tar -C ${layer} -czf $out . -'' + QEMU_OPTS = "-netdev user,id=net0 -device virtio-net-pci,netdev=net0"; + }); +in + pullImage diff --git a/pkgs/build-support/docker/pull.sh b/pkgs/build-support/docker/pull.sh index 7ba146e9de09..db643f5e88e5 100644 --- a/pkgs/build-support/docker/pull.sh +++ b/pkgs/build-support/docker/pull.sh @@ -1,86 +1,36 @@ -# Reference: docker src contrib/download-frozen-image.sh - source $stdenv/setup -# Curl flags to handle redirects, not use EPSV, handle cookies for -# servers to need them during redirects, and work on SSL without a -# certificate (this isn't a security problem because we check the -# cryptographic hash of the output anyway). -curl=$(command -v curl) -curl() { - [[ -n ${token:-} ]] && set -- -H "Authorization: Token $token" "$@" - $curl \ - --location --max-redirs 20 \ - --retry 3 \ - --fail \ - --disable-epsv \ - --cookie-jar cookies \ - --insecure \ - $curlOpts \ - $NIX_CURL_FLAGS \ - "$@" -} - -fetchLayer() { - local url="$1" - local dest="$2" - local curlexit=18; - - # if we get error code 18, resume partial download - while [ $curlexit -eq 18 ]; do - # keep this inside an if statement, since on failure it doesn't abort the script - if curl -C - "$url" --output "$dest"; then - return 0 - else - curlexit=$?; - fi - done - - return $curlexit -} - -headers=$(curl -o /dev/null -D- -H 'X-Docker-Token: true' \ - "$indexUrl/$registryVersion/repositories/$imageName/images") +mkdir -p /var/lib/docker +mkfs.ext4 /dev/vda +mount -t ext4 /dev/vda /var/lib/docker -header() { - grep $1 <<< "$headers" | tr -d '\r' | cut -d ' ' -f 2 -} +modprobe virtio_net +dhclient eth0 -# this only takes the first endpoint, more may be provided -# https://docs.docker.com/v1.6/reference/api/docker-io_api/ -if ! registryUrl=$(header X-Docker-Endpoints); then - echo "error: index returned no endpoint" - exit 1 -fi -baseUrl="https://$registryUrl/$registryVersion" +mkdir -p /etc/ssl/certs/ +cp "$certs" "/etc/ssl/certs/" -token="$(header X-Docker-Token || true)"; - -if [ -z "$imageId" ]; then - imageId="$(curl "$baseUrl/repositories/$imageName/tags/$imageTag")" - imageId="${imageId//\"/}" - if [ -z "$imageId" ]; then - echo "error: no image ID found for ${imageName}:${imageTag}" - exit 1 +# from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount +mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup +cd /sys/fs/cgroup +for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do + mkdir -p $sys + if ! mountpoint -q $sys; then + if ! mount -n -t cgroup -o $sys cgroup $sys; then + rmdir $sys || true fi + fi +done - echo "found image ${imageName}:${imageTag}@$imageId" -fi - -mkdir -p $out +# run docker daemon +dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock & -jshon -n object \ - -n object -s "$imageId" -i "$imageTag" \ - -i "$imageName" > $out/repositories +until $(curl --output /dev/null --silent --connect-timeout 2 http://127.0.0.1:5555); do + printf '.' + sleep 1 +done -curl "$baseUrl/images/$imageId/ancestry" -o ancestry.json +rm -r $out -layerIds=$(jshon -a -u < ancestry.json) -for layerId in $layerIds; do - echo "fetching layer $layerId" - - mkdir "$out/$layerId" - echo '1.0' > "$out/$layerId/VERSION" - curl "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json" - fetchLayer "$baseUrl/images/$layerId/layer" "$out/$layerId/layer.tar" -done +docker pull ${imageId} +docker save ${imageId} > $out -- cgit 1.4.1