Merge branch 'master' into docker-dirlinks

4 files changed, 161 insertions, 134 deletions

diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
index 8230eb69c29c..b9d9fd2a54c8 100644
--- a/pkgs/build-support/docker/default.nix
+++ b/pkgs/build-support/docker/default.nix
@@ -6,9 +6,12 @@
   findutils,
   go,
   jshon,
+  jq,
   lib,
   pkgs,
   pigz,
+  nixUnstable,
+  perl,
   runCommand,
   rsync,
   shadow,
@@ -26,7 +29,7 @@
 rec {
 
   examples = import ./examples.nix {
-    inherit pkgs buildImage pullImage shadowSetup;
+    inherit pkgs buildImage pullImage shadowSetup buildImageWithNixDb;
   };
 
   pullImage = callPackage ./pull.nix {};
@@ -42,7 +45,7 @@ rec {
     cp ${./tarsum.go} tarsum.go
     export GOPATH=$(pwd)
     mkdir src
-    ln -sT ${docker.src}/pkg/tarsum src/tarsum
+    ln -sT ${docker.src}/components/engine/pkg/tarsum src/tarsum
     go build
 
     cp tarsum $out
@@ -82,7 +85,7 @@ rec {
     export PATH=${shadow}/bin:$PATH
     mkdir -p /etc/pam.d
     if [[ ! -f /etc/passwd ]]; then
-      echo "root:x:0:0::/root:/bin/sh" > /etc/passwd
+      echo "root:x:0:0::/root:${stdenv.shell}" > /etc/passwd
       echo "root:!x:::::::" > /etc/shadow
     fi
     if [[ ! -f /etc/group ]]; then
@@ -225,6 +228,19 @@ rec {
       ${text}
     '';
 
+  nixRegistration = contents: runCommand "nix-registration" {
+    buildInputs = [ nixUnstable perl ];
+    # For obtaining the closure of `contents'.
+    exportReferencesGraph =
+      let contentsList = if builtins.isList contents then contents else [ contents ];
+      in map (x: [("closure-" + baseNameOf x) x]) contentsList;
+    }
+    ''
+      mkdir $out
+      printRegistration=1 perl ${pkgs.pathsFromGraph} closure-* > $out/db.dump
+      perl ${pkgs.pathsFromGraph} closure-* > $out/storePaths
+    '';
+
   # Create a "layer" (set of files).
   mkPureLayer = {
     # Name of the layer
@@ -238,11 +254,10 @@ rec {
     # into directories.
     keepContentsDirlinks ? false,
     # Additional commands to run on the layer before it is tar'd up.
-    extraCommands ? ""
+    extraCommands ? "", uid ? 0, gid ? 0
   }:
     runCommand "docker-layer-${name}" {
       inherit baseJson contents extraCommands;
-
       buildInputs = [ jshon rsync ];
     }
     ''
@@ -257,6 +272,8 @@ rec {
         echo "No contents to add to layer."
       fi
 
+      chmod ug+w layer
+
       if [[ -n $extraCommands ]]; then
         (cd layer; eval "$extraCommands")
       fi
@@ -264,7 +281,7 @@ rec {
       # Tar up the layer and throw it into 'layer.tar'.
       echo "Packing layer..."
       mkdir $out
-      tar -C layer --mtime="@$SOURCE_DATE_EPOCH" -cf $out/layer.tar .
+      tar -C layer --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar .
 
       # Compute a checksum of the tarball.
       echo "Computing layer checksum..."
@@ -320,6 +337,8 @@ rec {
           echo "Adding $item..."
           rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/
         done
+
+        chmod ug+w layer
       '';
 
       postMount = ''
@@ -387,11 +406,13 @@ rec {
     # Docker config; e.g. what command to run on the container.
     config ? null,
     # Optional bash script to run on the files prior to fixturizing the layer.
-    extraCommands ? "",
+    extraCommands ? "", uid ? 0, gid ? 0,
     # Optional bash script to run as root on the image when provisioning.
     runAsRoot ? null,
     # Size of the virtual machine disk to provision when building the image.
     diskSize ? 1024,
+    # Time of creation of the image.
+    created ? "1970-01-01T00:00:01Z",
   }:
 
     let
@@ -399,17 +420,16 @@ rec {
 
       # Create a JSON blob of the configuration. Set the date to unix zero.
       baseJson = writeText "${baseName}-config.json" (builtins.toJSON {
-        created = "1970-01-01T00:00:01Z";
+        inherit created config;
         architecture = "amd64";
         os = "linux";
-        config = config;
       });
 
       layer =
         if runAsRoot == null
         then mkPureLayer {
           name = baseName;
-          inherit baseJson contents keepContentsDirlinks extraCommands;
+          inherit baseJson contents keepContentsDirlinks extraCommands uid gid;
         } else mkRootLayer {
           name = baseName;
           inherit baseJson fromImage fromImageName fromImageTag
@@ -417,9 +437,10 @@ rec {
                   extraCommands;
         };
       result = runCommand "docker-image-${baseName}.tar.gz" {
-        buildInputs = [ jshon pigz coreutils findutils ];
-        imageName = name;
-        imageTag = tag;
+        buildInputs = [ jshon pigz coreutils findutils jq ];
+        # Image name and tag must be lowercase
+        imageName = lib.toLower name;
+        imageTag = lib.toLower tag;
         inherit fromImage baseJson;
         layerClosure = writeReferencesToFile layer;
         passthru.buildArgs = args;
@@ -443,6 +464,9 @@ rec {
         if [[ -n "$fromImage" ]]; then
           echo "Unpacking base image..."
           tar -C image -xpf "$fromImage"
+          # Do not import the base image configuration and manifest
+          chmod a+w image image/*.json
+          rm -f image/*.json
 
           if [[ -z "$fromImageName" ]]; then
             fromImageName=$(jshon -k < image/repositories|head -n1)
@@ -501,6 +525,24 @@ rec {
         # Use the temp folder we've been working on to create a new image.
         mv temp image/$layerID
 
+        # Create image json and image manifest
+        imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}")
+        manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]")
+        currentID=$layerID
+        while [[ -n "$currentID" ]]; do
+          layerChecksum=$(sha256sum image/$currentID/layer.tar | cut -d ' ' -f1)
+          imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .")
+          imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .")
+          manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$currentID/layer.tar\"] + .")
+
+          currentID=$(cat image/$currentID/json | (jshon -e parent -u 2>/dev/null || true))
+        done
+
+        imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1)
+        echo "$imageJson" > "image/$imageJsonChecksum.json"
+        manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"$imageJsonChecksum.json\"")
+        echo "$manifestJson" > image/manifest.json
+
         # Store the json under the name image/repositories.
         jshon -n object \
           -n object -s "$layerID" -i "$imageTag" \
@@ -510,11 +552,44 @@ rec {
         chmod -R a-w image
 
         echo "Cooking the image..."
-        tar -C image --mtime="@$SOURCE_DATE_EPOCH" -c . | pigz -nT > $out
+        tar -C image --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'./':: -c . | pigz -nT > $out
 
         echo "Finished."
       '';
 
     in
     result;
+
+  # Build an image and populate its nix database with the provided
+  # contents. The main purpose is to be able to use nix commands in
+  # the container.
+  # Be careful since this doesn't work well with multilayer.
+  buildImageWithNixDb = args@{ contents ? null, extraCommands ? "", ... }:
+    buildImage (args // {
+      extraCommands = ''
+        echo "Generating the nix database..."
+        echo "Warning: only the database of the deepest Nix layer is loaded."
+        echo "         If you want to use nix commands in the container, it would"
+        echo "         be better to only have one layer that contains a nix store."
+        # This requires Nix 1.12 or higher
+        export NIX_REMOTE=local?root=$PWD
+        ${nixUnstable}/bin/nix-store --load-db < ${nixRegistration contents}/db.dump
+
+        # We fill the store in order to run the 'verify' command that
+        # generates hash and size of output paths.
+        # Note when Nix 1.12 is be the stable one, the database dump
+        # generated by the exportReferencesGraph function will
+        # contains sha and size. See
+        # https://github.com/NixOS/nix/commit/c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a
+        storePaths=$(cat ${nixRegistration contents}/storePaths)
+        echo "Copying everything to /nix/store (will take a while)..."
+        cp -prd $storePaths nix/store/
+        ${nixUnstable}/bin/nix-store --verify --check-contents
+
+        mkdir -p nix/var/nix/gcroots/docker/
+        for i in ${lib.concatStringsSep " " contents}; do
+          ln -s $i nix/var/nix/gcroots/docker/$(basename $i)
+        done;
+      '' + extraCommands;
+    });
 }
diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix
index 05b4a9b4f2d2..b9a334971744 100644
--- a/pkgs/build-support/docker/examples.nix
+++ b/pkgs/build-support/docker/examples.nix
@@ -7,7 +7,7 @@
 #  $ nix-build '<nixpkgs>' -A dockerTools.examples.redis
 #  $ docker load < result
 
-{ pkgs, buildImage, pullImage, shadowSetup }:
+{ pkgs, buildImage, pullImage, shadowSetup, buildImageWithNixDb }:
 
 rec {
   # 1. basic example
@@ -83,16 +83,12 @@ rec {
   };
 
   # 4. example of pulling an image. could be used as a base for other images
-  #
-  # ***** Currently broken, getting 404s. Perhaps the docker API has changed?
-  #
-  #
-  # debian = pullImage {
-  #   imageName = "debian";
-  #   imageTag = "jessie";
-  #   # this hash will need change if the tag is updated at docker hub
-  #   sha256 = "18kd495lc2k35h03bpcbdjnix17nlqbwf6nmq3sb161blf0dk14q";
-  # };
+  nixFromDockerHub = pullImage {
+    imageName = "nixos/nix";
+    imageTag = "1.11";
+    # this hash will need change if the tag is updated at docker hub
+    sha256 = "0nncn9pn5miygan51w34c2p9qssi96jgsaqv44dxxdprc8pg0g83";
+  };
 
   # 5. example of multiple contents, emacs and vi happily coexisting
   editors = buildImage {
@@ -105,4 +101,19 @@ rec {
       pkgs.nano
     ];
   };
+
+  # 5. nix example to play with the container nix store
+  # docker run -it --rm nix nix-store -qR $(nix-build '<nixpkgs>' -A nix)
+  nix = buildImageWithNixDb {
+    name = "nix";
+    contents = [
+      # nix-store uses cat program to display results as specified by
+      # the image env variable NIX_PAGER.
+      pkgs.coreutils
+      pkgs.nix
+    ];
+    config = {
+      Env = [ "NIX_PAGER=cat" ];
+    };
+  };
 }
diff --git a/pkgs/build-support/docker/pull.nix b/pkgs/build-support/docker/pull.nix
index 0e1b147f6e18..5ccd0a41c5e4 100644
--- a/pkgs/build-support/docker/pull.nix
+++ b/pkgs/build-support/docker/pull.nix
@@ -1,41 +1,32 @@
-{ stdenv, lib, curl, jshon, python, runCommand }:
-
-# Inspired and simplified version of fetchurl.
+{ stdenv, lib, docker, vmTools, utillinux, curl, kmod, dhcp, cacert, e2fsprogs }:
+let
+  nameReplace = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name;
+in
 # For simplicity we only support sha256.
+{ imageName, imageTag ? "latest", imageId ? "${imageName}:${imageTag}"
+, sha256, name ? (nameReplace "docker-image-${imageName}-${imageTag}.tar") }:
+let
+  pullImage = vmTools.runInLinuxVM (
+    stdenv.mkDerivation {
+      inherit name imageId;
 
-# Currently only registry v1 is supported, compatible with Docker Hub.
-
-{ imageName, imageTag ? "latest", imageId ? null
-, sha256, name ? "${imageName}-${imageTag}"
-, indexUrl ? "https://index.docker.io"
-, registryVersion ? "v1"
-, curlOpts ? "" }:
-
-assert registryVersion == "v1";
-
-let layer = stdenv.mkDerivation {
-  inherit name imageName imageTag imageId
-          indexUrl registryVersion curlOpts;
+      certs = "${cacert}/etc/ssl/certs/ca-bundle.crt";
 
-  builder = ./pull.sh;
-  detjson = ./detjson.py;
+      builder = ./pull.sh;
 
-  buildInputs = [ curl jshon python ];
+      buildInputs = [ curl utillinux docker kmod dhcp cacert e2fsprogs ];
 
-  outputHashAlgo = "sha256";
-  outputHash = sha256;
-  outputHashMode = "recursive";
+      outputHashAlgo = "sha256";
+      outputHash = sha256;
 
-  impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [
-    # This variable allows the user to pass additional options to curl
-    "NIX_CURL_FLAGS"
-  ];
+      impureEnvVars = lib.fetchers.proxyImpureEnvVars;
 
-  # Doing the download on a remote machine just duplicates network
-  # traffic, so don't do that.
-  preferLocalBuild = true;
-};
+      preVM = vmTools.createEmptyImage {
+        size = 2048;
+        fullName = "${name}-disk";
+      };
 
-in runCommand "${name}.tar.gz" {} ''
-  tar -C ${layer} -czf $out .
-''
+      QEMU_OPTS = "-netdev user,id=net0 -device virtio-net-pci,netdev=net0";
+    });
+in
+  pullImage
diff --git a/pkgs/build-support/docker/pull.sh b/pkgs/build-support/docker/pull.sh
index 7ba146e9de09..0b1e9f310ee9 100644
--- a/pkgs/build-support/docker/pull.sh
+++ b/pkgs/build-support/docker/pull.sh
@@ -1,86 +1,36 @@
-# Reference: docker src contrib/download-frozen-image.sh
-
 source $stdenv/setup
 
-# Curl flags to handle redirects, not use EPSV, handle cookies for
-# servers to need them during redirects, and work on SSL without a
-# certificate (this isn't a security problem because we check the
-# cryptographic hash of the output anyway).
-curl=$(command -v curl)
-curl() {
-  [[ -n ${token:-} ]] && set -- -H "Authorization: Token $token" "$@"
-  $curl \
-    --location --max-redirs 20 \
-    --retry 3 \
-    --fail \
-    --disable-epsv \
-    --cookie-jar cookies \
-    --insecure \
-    $curlOpts \
-    $NIX_CURL_FLAGS \
-    "$@"
-}
-
-fetchLayer() {
-    local url="$1"
-    local dest="$2"
-    local curlexit=18;
-
-    # if we get error code 18, resume partial download
-    while [ $curlexit -eq 18 ]; do
-        # keep this inside an if statement, since on failure it doesn't abort the script
-        if curl -C - "$url" --output "$dest"; then
-            return 0
-        else
-            curlexit=$?;
-        fi
-    done
-
-    return $curlexit
-}
-
-headers=$(curl -o /dev/null -D- -H 'X-Docker-Token: true' \
-          "$indexUrl/$registryVersion/repositories/$imageName/images")
+mkdir -p /var/lib/docker
+mkfs.ext4 /dev/vda
+mount -t ext4 /dev/vda /var/lib/docker
 
-header() {
-  grep $1 <<< "$headers" | tr -d '\r' | cut -d ' ' -f 2
-}
+modprobe virtio_net
+dhclient eth0
 
-# this only takes the first endpoint, more may be provided
-# https://docs.docker.com/v1.6/reference/api/docker-io_api/
-if ! registryUrl=$(header X-Docker-Endpoints); then
-  echo "error: index returned no endpoint"
-  exit 1
-fi
-baseUrl="https://$registryUrl/$registryVersion"
+mkdir -p /etc/ssl/certs/
+cp "$certs" "/etc/ssl/certs/"
 
-token="$(header X-Docker-Token || true)";
-
-if [ -z "$imageId" ]; then
-    imageId="$(curl "$baseUrl/repositories/$imageName/tags/$imageTag")"
-    imageId="${imageId//\"/}"
-    if [ -z "$imageId" ]; then
-        echo "error: no image ID found for ${imageName}:${imageTag}"
-        exit 1
+# from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount
+mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
+cd /sys/fs/cgroup
+for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
+  mkdir -p $sys
+  if ! mountpoint -q $sys; then
+    if ! mount -n -t cgroup -o $sys cgroup $sys; then
+      rmdir $sys || true
     fi
+  fi
+done
 
-    echo "found image ${imageName}:${imageTag}@$imageId"
-fi
-
-mkdir -p $out
+# run docker daemon
+dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock &
 
-jshon -n object \
-  -n object -s "$imageId" -i "$imageTag" \
-  -i "$imageName" > $out/repositories
+until docker ps 2>/dev/null; do
+  printf '.'
+  sleep 1
+done
 
-curl "$baseUrl/images/$imageId/ancestry" -o ancestry.json
+rm -r $out
 
-layerIds=$(jshon -a -u < ancestry.json)
-for layerId in $layerIds; do
-    echo "fetching layer $layerId"
-    
-    mkdir "$out/$layerId"
-    echo '1.0' > "$out/$layerId/VERSION"
-    curl "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json"
-    fetchLayer "$baseUrl/images/$layerId/layer" "$out/$layerId/layer.tar"
-done
+docker pull ${imageId}
+docker save ${imageId} > $out