nixos/dockerPreloader: preload docker images (#49379)

This module permits to preload Docker image in a VM in order to reduce
OIs on file copies. This module has to be only used in testing
environments, when the test requires several Docker images such as in
Kubernetes tests. In this case,
`virtualisation.dockerPreloader.images` can replace the
`services.kubernetes.kubelet.seedDockerImages` options.

The idea is to populate the /var/lib/docker directory by mounting qcow
files (we uses qcow file to avoid permission issues) that contain images.

For each image specified in
config.virtualisation.dockerPreloader.images:
1. The image is loaded by Docker in a VM
2. The resulting /var/lib/docker is written to a QCOW file

This set of QCOW files can then be used to populate the
/var/lib/docker:
1. Each QCOW is mounted in the VM
2. Symlink are created from these mount points to /var/lib/docker
3. A /var/lib/docker/image/overlay2/repositories.json file is generated
4. The docker daemon is started.

4 files changed, 167 insertions, 1 deletions

diff --git a/nixos/modules/virtualisation/docker-preloader.nix b/nixos/modules/virtualisation/docker-preloader.nix
new file mode 100644
index 000000000000..faa94f53d98f
--- /dev/null
+++ b/nixos/modules/virtualisation/docker-preloader.nix
@@ -0,0 +1,135 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+with builtins;
+
+let
+  cfg = config.virtualisation;
+
+  sanitizeImageName = image: replaceStrings ["/"] ["-"] image.imageName;
+  hash = drv: head (split "-" (baseNameOf drv.outPath));
+  # The label of an ext4 FS is limited to 16 bytes
+  labelFromImage = image: substring 0 16 (hash image);
+
+  # The Docker image is loaded and some files from /var/lib/docker/
+  # are written into a qcow image.
+  preload = image: pkgs.vmTools.runInLinuxVM (
+    pkgs.runCommand "docker-preload-image-${sanitizeImageName image}" {
+      buildInputs = with pkgs; [ docker e2fsprogs utillinux curl kmod ];
+      preVM = pkgs.vmTools.createEmptyImage {
+        size = cfg.dockerPreloader.qcowSize;
+        fullName = "docker-deamon-image.qcow2";
+      };
+    }
+    ''
+      mkfs.ext4 /dev/vda
+      e2label /dev/vda ${labelFromImage image}
+      mkdir -p /var/lib/docker
+      mount -t ext4 /dev/vda /var/lib/docker
+
+      modprobe overlay
+
+      # from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount
+      mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
+      cd /sys/fs/cgroup
+      for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
+        mkdir -p $sys
+        if ! mountpoint -q $sys; then
+          if ! mount -n -t cgroup -o $sys cgroup $sys; then
+            rmdir $sys || true
+          fi
+        fi
+      done
+
+      dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock &
+
+      until $(curl --output /dev/null --silent --connect-timeout 2 http://127.0.0.1:5555); do
+        printf '.'
+        sleep 1
+      done
+
+      docker load -i ${image}
+
+      kill %1
+      find /var/lib/docker/ -maxdepth 1 -mindepth 1 -not -name "image" -not -name "overlay2" | xargs rm -rf
+    '');
+
+  preloadedImages = map preload cfg.dockerPreloader.images;
+
+in
+
+{
+  options.virtualisation.dockerPreloader = {
+    images = mkOption {
+      default = [ ];
+      type = types.listOf types.package;
+      description =
+      ''
+        A list of Docker images to preload (in the /var/lib/docker directory).
+      '';
+    };
+    qcowSize = mkOption {
+      default = 1024;
+      type = types.int;
+      description =
+      ''
+        The size (MB) of qcow files.
+      '';
+    };
+  };
+
+  config = {
+    assertions = [{
+      # If docker.storageDriver is null, Docker choose the storage
+      # driver. So, in this case, we cannot be sure overlay2 is used.
+      assertion = cfg.dockerPreloader.images == []
+        || cfg.docker.storageDriver == "overlay2"
+        || cfg.docker.storageDriver == "overlay"
+        || cfg.docker.storageDriver == null;
+      message = "The Docker image Preloader only works with overlay2 storage driver!";
+    }];
+
+    virtualisation.qemu.options =
+      map (path: "-drive if=virtio,file=${path}/disk-image.qcow2,readonly,media=cdrom,format=qcow2")
+      preloadedImages;
+
+
+    # All attached QCOW files are mounted and their contents are linked
+    # to /var/lib/docker/ in order to make image available.
+    systemd.services.docker-preloader = {
+      description = "Preloaded Docker images";
+      wantedBy = ["docker.service"];
+      after = ["network.target"];
+      path = with pkgs; [ mount rsync jq ];
+      script = ''
+        mkdir -p /var/lib/docker/overlay2/l /var/lib/docker/image/overlay2
+        echo '{}' > /tmp/repositories.json
+
+        for i in ${concatStringsSep " " (map labelFromImage cfg.dockerPreloader.images)}; do
+          mkdir -p /mnt/docker-images/$i
+
+          # The ext4 label is limited to 16 bytes
+          mount /dev/disk/by-label/$(echo $i | cut -c1-16) -o ro,noload /mnt/docker-images/$i
+
+          find /mnt/docker-images/$i/overlay2/ -maxdepth 1 -mindepth 1 -not -name l\
+             -exec ln -s '{}' /var/lib/docker/overlay2/ \;
+          cp -P /mnt/docker-images/$i/overlay2/l/* /var/lib/docker/overlay2/l/
+
+          rsync -a /mnt/docker-images/$i/image/ /var/lib/docker/image/
+
+          # Accumulate image definitions
+          cp /tmp/repositories.json /tmp/repositories.json.tmp
+          jq -s '.[0] * .[1]' \
+            /tmp/repositories.json.tmp \
+            /mnt/docker-images/$i/image/overlay2/repositories.json \
+            > /tmp/repositories.json
+        done
+
+        mv /tmp/repositories.json /var/lib/docker/image/overlay2/repositories.json
+      '';
+      serviceConfig = {
+        Type = "oneshot";
+      };
+    };
+  };
+}
diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix
index 4e9c87222d0a..ed3431554be4 100644
--- a/nixos/modules/virtualisation/qemu-vm.nix
+++ b/nixos/modules/virtualisation/qemu-vm.nix
@@ -185,7 +185,10 @@ let
 in
 
 {
-  imports = [ ../profiles/qemu-guest.nix ];
+  imports = [
+    ../profiles/qemu-guest.nix
+   ./docker-preloader.nix
+  ];
 
   options = {
 
diff --git a/nixos/release.nix b/nixos/release.nix
index 51505d6aab9d..2bd70f7962fe 100644
--- a/nixos/release.nix
+++ b/nixos/release.nix
@@ -283,6 +283,7 @@ in rec {
   tests.docker-tools = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-tools.nix {};
   tests.docker-tools-overlay = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-tools-overlay.nix {};
   tests.docker-edge = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-edge.nix {};
+  tests.docker-preloader = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-preloader.nix {};
   tests.docker-registry = callTest tests/docker-registry.nix {};
   tests.dovecot = callTest tests/dovecot.nix {};
   tests.dnscrypt-proxy = callTestOnMatchingSystems ["x86_64-linux"] tests/dnscrypt-proxy.nix {};
diff --git a/nixos/tests/docker-preloader.nix b/nixos/tests/docker-preloader.nix
new file mode 100644
index 000000000000..eeedec9a392e
--- /dev/null
+++ b/nixos/tests/docker-preloader.nix
@@ -0,0 +1,27 @@
+import ./make-test.nix ({ pkgs, ...} : {
+  name = "docker-preloader";
+  meta = with pkgs.stdenv.lib.maintainers; {
+    maintainers = [ lewo ];
+  };
+
+  nodes = {
+    docker =
+      { pkgs, ... }:
+        {
+          virtualisation.docker.enable = true;
+          virtualisation.dockerPreloader.images = [ pkgs.dockerTools.examples.nix pkgs.dockerTools.examples.bash ];
+
+          services.openssh.enable = true;
+          services.openssh.permitRootLogin = "yes";
+          services.openssh.extraConfig = "PermitEmptyPasswords yes";
+          users.extraUsers.root.password = "";
+        };
+  };    
+  testScript = ''
+    startAll;
+    
+    $docker->waitForUnit("sockets.target");
+    $docker->succeed("docker run nix nix-store --version");
+    $docker->succeed("docker run bash bash --version");
+  '';
+})