From 3fb4eb1c432a4f8cc92965db0e7b2c2856bde596 Mon Sep 17 00:00:00 2001 From: lewo Date: Sat, 3 Nov 2018 01:00:53 +0100 Subject: nixos/dockerPreloader: preload docker images (#49379) This module permits to preload Docker image in a VM in order to reduce OIs on file copies. This module has to be only used in testing environments, when the test requires several Docker images such as in Kubernetes tests. In this case, `virtualisation.dockerPreloader.images` can replace the `services.kubernetes.kubelet.seedDockerImages` options. The idea is to populate the /var/lib/docker directory by mounting qcow files (we uses qcow file to avoid permission issues) that contain images. For each image specified in config.virtualisation.dockerPreloader.images: 1. The image is loaded by Docker in a VM 2. The resulting /var/lib/docker is written to a QCOW file This set of QCOW files can then be used to populate the /var/lib/docker: 1. Each QCOW is mounted in the VM 2. Symlink are created from these mount points to /var/lib/docker 3. A /var/lib/docker/image/overlay2/repositories.json file is generated 4. The docker daemon is started. --- nixos/modules/virtualisation/docker-preloader.nix | 135 ++++++++++++++++++++++ nixos/modules/virtualisation/qemu-vm.nix | 5 +- nixos/release.nix | 1 + nixos/tests/docker-preloader.nix | 27 +++++ 4 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 nixos/modules/virtualisation/docker-preloader.nix create mode 100644 nixos/tests/docker-preloader.nix (limited to 'nixos') diff --git a/nixos/modules/virtualisation/docker-preloader.nix b/nixos/modules/virtualisation/docker-preloader.nix new file mode 100644 index 000000000000..faa94f53d98f --- /dev/null +++ b/nixos/modules/virtualisation/docker-preloader.nix @@ -0,0 +1,135 @@ +{ config, lib, pkgs, ... }: + +with lib; +with builtins; + +let + cfg = config.virtualisation; + + sanitizeImageName = image: replaceStrings ["/"] ["-"] image.imageName; + hash = drv: head (split "-" (baseNameOf drv.outPath)); + # The label of an ext4 FS is limited to 16 bytes + labelFromImage = image: substring 0 16 (hash image); + + # The Docker image is loaded and some files from /var/lib/docker/ + # are written into a qcow image. + preload = image: pkgs.vmTools.runInLinuxVM ( + pkgs.runCommand "docker-preload-image-${sanitizeImageName image}" { + buildInputs = with pkgs; [ docker e2fsprogs utillinux curl kmod ]; + preVM = pkgs.vmTools.createEmptyImage { + size = cfg.dockerPreloader.qcowSize; + fullName = "docker-deamon-image.qcow2"; + }; + } + '' + mkfs.ext4 /dev/vda + e2label /dev/vda ${labelFromImage image} + mkdir -p /var/lib/docker + mount -t ext4 /dev/vda /var/lib/docker + + modprobe overlay + + # from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount + mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup + cd /sys/fs/cgroup + for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do + mkdir -p $sys + if ! mountpoint -q $sys; then + if ! mount -n -t cgroup -o $sys cgroup $sys; then + rmdir $sys || true + fi + fi + done + + dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock & + + until $(curl --output /dev/null --silent --connect-timeout 2 http://127.0.0.1:5555); do + printf '.' + sleep 1 + done + + docker load -i ${image} + + kill %1 + find /var/lib/docker/ -maxdepth 1 -mindepth 1 -not -name "image" -not -name "overlay2" | xargs rm -rf + ''); + + preloadedImages = map preload cfg.dockerPreloader.images; + +in + +{ + options.virtualisation.dockerPreloader = { + images = mkOption { + default = [ ]; + type = types.listOf types.package; + description = + '' + A list of Docker images to preload (in the /var/lib/docker directory). + ''; + }; + qcowSize = mkOption { + default = 1024; + type = types.int; + description = + '' + The size (MB) of qcow files. + ''; + }; + }; + + config = { + assertions = [{ + # If docker.storageDriver is null, Docker choose the storage + # driver. So, in this case, we cannot be sure overlay2 is used. + assertion = cfg.dockerPreloader.images == [] + || cfg.docker.storageDriver == "overlay2" + || cfg.docker.storageDriver == "overlay" + || cfg.docker.storageDriver == null; + message = "The Docker image Preloader only works with overlay2 storage driver!"; + }]; + + virtualisation.qemu.options = + map (path: "-drive if=virtio,file=${path}/disk-image.qcow2,readonly,media=cdrom,format=qcow2") + preloadedImages; + + + # All attached QCOW files are mounted and their contents are linked + # to /var/lib/docker/ in order to make image available. + systemd.services.docker-preloader = { + description = "Preloaded Docker images"; + wantedBy = ["docker.service"]; + after = ["network.target"]; + path = with pkgs; [ mount rsync jq ]; + script = '' + mkdir -p /var/lib/docker/overlay2/l /var/lib/docker/image/overlay2 + echo '{}' > /tmp/repositories.json + + for i in ${concatStringsSep " " (map labelFromImage cfg.dockerPreloader.images)}; do + mkdir -p /mnt/docker-images/$i + + # The ext4 label is limited to 16 bytes + mount /dev/disk/by-label/$(echo $i | cut -c1-16) -o ro,noload /mnt/docker-images/$i + + find /mnt/docker-images/$i/overlay2/ -maxdepth 1 -mindepth 1 -not -name l\ + -exec ln -s '{}' /var/lib/docker/overlay2/ \; + cp -P /mnt/docker-images/$i/overlay2/l/* /var/lib/docker/overlay2/l/ + + rsync -a /mnt/docker-images/$i/image/ /var/lib/docker/image/ + + # Accumulate image definitions + cp /tmp/repositories.json /tmp/repositories.json.tmp + jq -s '.[0] * .[1]' \ + /tmp/repositories.json.tmp \ + /mnt/docker-images/$i/image/overlay2/repositories.json \ + > /tmp/repositories.json + done + + mv /tmp/repositories.json /var/lib/docker/image/overlay2/repositories.json + ''; + serviceConfig = { + Type = "oneshot"; + }; + }; + }; +} diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index 4e9c87222d0a..ed3431554be4 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -185,7 +185,10 @@ let in { - imports = [ ../profiles/qemu-guest.nix ]; + imports = [ + ../profiles/qemu-guest.nix + ./docker-preloader.nix + ]; options = { diff --git a/nixos/release.nix b/nixos/release.nix index 51505d6aab9d..2bd70f7962fe 100644 --- a/nixos/release.nix +++ b/nixos/release.nix @@ -283,6 +283,7 @@ in rec { tests.docker-tools = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-tools.nix {}; tests.docker-tools-overlay = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-tools-overlay.nix {}; tests.docker-edge = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-edge.nix {}; + tests.docker-preloader = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-preloader.nix {}; tests.docker-registry = callTest tests/docker-registry.nix {}; tests.dovecot = callTest tests/dovecot.nix {}; tests.dnscrypt-proxy = callTestOnMatchingSystems ["x86_64-linux"] tests/dnscrypt-proxy.nix {}; diff --git a/nixos/tests/docker-preloader.nix b/nixos/tests/docker-preloader.nix new file mode 100644 index 000000000000..eeedec9a392e --- /dev/null +++ b/nixos/tests/docker-preloader.nix @@ -0,0 +1,27 @@ +import ./make-test.nix ({ pkgs, ...} : { + name = "docker-preloader"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ lewo ]; + }; + + nodes = { + docker = + { pkgs, ... }: + { + virtualisation.docker.enable = true; + virtualisation.dockerPreloader.images = [ pkgs.dockerTools.examples.nix pkgs.dockerTools.examples.bash ]; + + services.openssh.enable = true; + services.openssh.permitRootLogin = "yes"; + services.openssh.extraConfig = "PermitEmptyPasswords yes"; + users.extraUsers.root.password = ""; + }; + }; + testScript = '' + startAll; + + $docker->waitForUnit("sockets.target"); + $docker->succeed("docker run nix nix-store --version"); + $docker->succeed("docker run bash bash --version"); + ''; +}) -- cgit 1.4.1