diff options
Diffstat (limited to 'pkgs/build-support/docker')
| -rw-r--r-- | pkgs/build-support/docker/default.nix | 189 | ||||
| -rw-r--r-- | pkgs/build-support/docker/examples.nix | 7 | ||||
| -rwxr-xr-x | pkgs/build-support/docker/store-path-to-layer.sh | 24 |
3 files changed, 214 insertions, 6 deletions
diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 0cee1dd2916f..73639a521b6a 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -1,4 +1,5 @@ { + symlinkJoin, coreutils, docker, e2fsprogs, @@ -19,6 +20,7 @@ utillinux, vmTools, writeReferencesToFile, + referencesByPopularity, writeScript, writeText, }: @@ -77,7 +79,9 @@ rec { ln -sT ${docker.src}/components/engine/pkg/tarsum src/github.com/docker/docker/pkg/tarsum go build - cp tarsum $out + mkdir -p $out/bin + + cp tarsum $out/bin/ ''; # buildEnv creates symlinks to dirs, which is hard to edit inside the overlay VM @@ -270,6 +274,81 @@ rec { perl ${pkgs.pathsFromGraph} closure-* > $out/storePaths ''; + # Create $maxLayers worth of Docker Layers, one layer per store path + # unless there are more paths than $maxLayers. In that case, create + # $maxLayers-1 for the most popular layers, and smush the remainaing + # store paths in to one final layer. + mkManyPureLayers = { + name, + # Files to add to the layer. + closure, + configJson, + # Docker has a 42-layer maximum, we pick 24 to ensure there is plenty + # of room for extension + maxLayers ? 24 + }: + runCommand "${name}-granular-docker-layers" { + inherit maxLayers; + paths = referencesByPopularity closure; + buildInputs = [ jshon rsync tarsum ]; + enableParallelBuilding = true; + } + '' + # Delete impurities for store path layers, so they don't get + # shared and taint other projects. + cat ${configJson} \ + | jshon -d config \ + | jshon -s "1970-01-01T00:00:01Z" -i created > generic.json + + # WARNING! + # The following code is fiddly w.r.t. ensuring every layer is + # created, and that no paths are missed. If you change the + # following head and tail call lines, double-check that your + # code behaves properly when the number of layers equals: + # maxLayers-1, maxLayers, and maxLayers+1 + head -n $((maxLayers - 1)) $paths | cat -n | xargs -P$NIX_BUILD_CORES -n2 ${./store-path-to-layer.sh} + if [ $(cat $paths | wc -l) -ge $maxLayers ]; then + tail -n+$maxLayers $paths | xargs ${./store-path-to-layer.sh} $maxLayers + fi + + echo "Finished building layer '$name'" + + mv ./layers $out + ''; + + # Create a "Customisation" layer which adds symlinks at the root of + # the image to the root paths of the closure. Also add the config + # data like what command to run and the environment to run it in. + mkCustomisationLayer = { + name, + # Files to add to the layer. + contents, + baseJson, + uid ? 0, gid ? 0, + }: + runCommand "${name}-customisation-layer" { + buildInputs = [ jshon rsync tarsum ]; + } + '' + cp -r ${contents}/ ./layer + + # Tar up the layer and throw it into 'layer.tar'. + echo "Packing layer..." + mkdir $out + tar -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar . + + # Compute a checksum of the tarball. + echo "Computing layer checksum..." + tarhash=$(tarsum < $out/layer.tar) + + # Add a 'checksum' field to the JSON, with the value set to the + # checksum of the tarball. + cat ${baseJson} | jshon -s "$tarhash" -i checksum > $out/json + + # Indicate to docker that we're using schema version 1.0. + echo -n "1.0" > $out/VERSION + ''; + # Create a "layer" (set of files). mkPureLayer = { # Name of the layer @@ -287,7 +366,7 @@ rec { }: runCommand "docker-layer-${name}" { inherit baseJson contents extraCommands; - buildInputs = [ jshon rsync ]; + buildInputs = [ jshon rsync tarsum ]; } '' mkdir layer @@ -314,11 +393,11 @@ rec { # Compute a checksum of the tarball. echo "Computing layer checksum..." - tarsum=$(${tarsum} < $out/layer.tar) + tarhash=$(tarsum < $out/layer.tar) # Add a 'checksum' field to the JSON, with the value set to the # checksum of the tarball. - cat ${baseJson} | jshon -s "$tarsum" -i checksum > $out/json + cat ${baseJson} | jshon -s "$tarhash" -i checksum > $out/json # Indicate to docker that we're using schema version 1.0. echo -n "1.0" > $out/VERSION @@ -402,8 +481,8 @@ rec { # Compute the tar checksum and add it to the output json. echo "Computing checksum..." - ts=$(${tarsum} < $out/layer.tar) - cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json + tarhash=$(${tarsum}/bin/tarsum < $out/layer.tar) + cat ${baseJson} | jshon -s "$tarhash" -i checksum > $out/json # Indicate to docker that we're using schema version 1.0. echo -n "1.0" > $out/VERSION @@ -411,6 +490,104 @@ rec { ''; }; + buildLayeredImage = { + # Image Name + name, + # Image tag, the Nix's output hash will be used if null + tag ? null, + # Files to put on the image (a nix store path or list of paths). + contents ? [], + # Docker config; e.g. what command to run on the container. + config ? {}, + # Time of creation of the image. Passing "now" will make the + # created date be the time of building. + created ? "1970-01-01T00:00:01Z", + # Docker's lowest maximum layer limit is 42-layers for an old + # version of the AUFS graph driver. We pick 24 to ensure there is + # plenty of room for extension. I believe the actual maximum is + # 128. + maxLayers ? 24 + }: + let + uid = 0; + gid = 0; + baseName = baseNameOf name; + contentsEnv = symlinkJoin { name = "bulk-layers"; paths = (if builtins.isList contents then contents else [ contents ]); }; + + configJson = let + pure = writeText "${baseName}-config.json" (builtins.toJSON { + inherit created config; + architecture = "amd64"; + os = "linux"; + }); + impure = runCommand "${baseName}-standard-dynamic-date.json" + { buildInputs = [ jq ]; } + '' + jq ".created = \"$(TZ=utc date --iso-8601="seconds")\"" ${pure} > $out + ''; + in if created == "now" then impure else pure; + + bulkLayers = mkManyPureLayers { + name = baseName; + closure = writeText "closure" "${contentsEnv} ${configJson}"; + # One layer will be taken up by the customisationLayer, so + # take up one less. + maxLayers = maxLayers - 1; + inherit configJson; + }; + customisationLayer = mkCustomisationLayer { + name = baseName; + contents = contentsEnv; + baseJson = configJson; + inherit uid gid; + }; + result = runCommand "docker-image-${baseName}.tar.gz" { + buildInputs = [ jshon pigz coreutils findutils jq ]; + # Image name and tag must be lowercase + imageName = lib.toLower name; + imageTag = if tag == null then "" else lib.toLower tag; + baseJson = configJson; + } '' + ${lib.optionalString (tag == null) '' + outName="$(basename "$out")" + outHash=$(echo "$outName" | cut -d - -f 1) + + imageTag=$outHash + ''} + + find ${bulkLayers} -mindepth 1 -maxdepth 1 | sort -t/ -k5 -n > layer-list + echo ${customisationLayer} >> layer-list + + mkdir image + imageJson=$(cat ${configJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}") + manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]") + for layer in $(cat layer-list); do + layerChecksum=$(sha256sum $layer/layer.tar | cut -d ' ' -f1) + layerID=$(sha256sum "$layer/json" | cut -d ' ' -f 1) + ln -s "$layer" "./image/$layerID" + + manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$layerID/layer.tar\"] + .") + imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"$(jq -r .created ${configJson})\"}] + .") + imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .") + done + imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1) + echo "$imageJson" > "image/$imageJsonChecksum.json" + manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"$imageJsonChecksum.json\"") + echo "$manifestJson" > image/manifest.json + + jshon -n object \ + -n object -s "$layerID" -i "$imageTag" \ + -i "$imageName" > image/repositories + + echo "Cooking the image..." + tar -C image --dereference --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --mode=a-w --xform s:'^./':: -c . | pigz -nT > $out + + echo "Finished." + ''; + + in + result; + # 1. extract the base image # 2. create the layer # 3. add layer deps to the layer itself, diffing with the base image diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index 822e0dbb31f2..003e7429a81b 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -150,4 +150,11 @@ rec { contents = [ pkgs.coreutils ]; created = "now"; }; + + # 10. Create a layered image + layered-image = pkgs.dockerTools.buildLayeredImage { + name = "layered-image"; + tag = "latest"; + config.Cmd = [ "${pkgs.hello}/bin/hello" ]; + }; } diff --git a/pkgs/build-support/docker/store-path-to-layer.sh b/pkgs/build-support/docker/store-path-to-layer.sh new file mode 100755 index 000000000000..ff814c1f6130 --- /dev/null +++ b/pkgs/build-support/docker/store-path-to-layer.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +set -eu + +layerNumber=$1 +shift + +layerPath="./layers/$layerNumber" +echo "Creating layer #$layerNumber for $@" + +mkdir -p "$layerPath" +tar -rpf "$layerPath/layer.tar" --hard-dereference --sort=name \ + --mtime="@$SOURCE_DATE_EPOCH" \ + --owner=0 --group=0 "$@" + +# Compute a checksum of the tarball. +tarhash=$(tarsum < $layerPath/layer.tar) + +# Add a 'checksum' field to the JSON, with the value set to the +# checksum of the tarball. +cat ./generic.json | jshon -s "$tarhash" -i checksum > $layerPath/json + +# Indicate to docker that we're using schema version 1.0. +echo -n "1.0" > $layerPath/VERSION |