Merge pull request #89540 from Patryk27/fixes/lxd-lxcfs

Fix `lxd`, so that it works with `lxcfs`

4 files changed, 201 insertions, 3 deletions

diff --git a/nixos/modules/virtualisation/lxd.nix b/nixos/modules/virtualisation/lxd.nix
index 53b89a9f55b1..3958fc2c1d7c 100644
--- a/nixos/modules/virtualisation/lxd.nix
+++ b/nixos/modules/virtualisation/lxd.nix
@@ -15,7 +15,6 @@ in
   ###### interface
 
   options = {
-
     virtualisation.lxd = {
       enable = mkOption {
         type = types.bool;
@@ -25,12 +24,18 @@ in
           containers. Users in the "lxd" group can interact with
           the daemon (e.g. to start or stop containers) using the
           <command>lxc</command> command line tool, among others.
+
+          Most of the time, you'll also want to start lxcfs, so
+          that containers can "see" the limits:
+          <code>
+            virtualisation.lxc.lxcfs.enable = true;
+          </code>
         '';
       };
 
       package = mkOption {
         type = types.package;
-        default = pkgs.lxd;
+        default = pkgs.lxd.override { nftablesSupport = config.networking.nftables.enable; };
         defaultText = "pkgs.lxd";
         description = ''
           The LXD package to use.
@@ -65,6 +70,7 @@ in
           with nixos.
         '';
       };
+
       recommendedSysctlSettings = mkOption {
         type = types.bool;
         default = false;
@@ -83,7 +89,6 @@ in
   ###### implementation
 
   config = mkIf cfg.enable {
-
     environment.systemPackages = [ cfg.package ];
 
     security.apparmor = {
@@ -115,6 +120,12 @@ in
         LimitNOFILE = "1048576";
         LimitNPROC = "infinity";
         TasksMax = "infinity";
+
+        # By default, `lxd` loads configuration files from hard-coded
+        # `/usr/share/lxc/config` - since this is a no-go for us, we have to
+        # explicitly tell it where the actual configuration files are
+        Environment = mkIf (config.virtualisation.lxc.lxcfs.enable)
+          "LXD_LXC_TEMPLATE_CONFIG=${pkgs.lxcfs}/share/lxc/config";
       };
     };
 
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 8e262d8eee7d..1bf091b361cb 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -178,6 +178,8 @@ in
   limesurvey = handleTest ./limesurvey.nix {};
   login = handleTest ./login.nix {};
   loki = handleTest ./loki.nix {};
+  lxd = handleTest ./lxd.nix {};
+  lxd-nftables = handleTest ./lxd-nftables.nix {};
   #logstash = handleTest ./logstash.nix {};
   lorri = handleTest ./lorri/default.nix {};
   magnetico = handleTest ./magnetico.nix {};
diff --git a/nixos/tests/lxd-nftables.nix b/nixos/tests/lxd-nftables.nix
new file mode 100644
index 000000000000..25517914db85
--- /dev/null
+++ b/nixos/tests/lxd-nftables.nix
@@ -0,0 +1,50 @@
+# This test makes sure that lxd stops implicitly depending on iptables when
+# user enabled nftables.
+#
+# It has been extracted from `lxd.nix` for clarity, and because switching from
+# iptables to nftables requires a full reboot, which is a bit hard inside NixOS
+# tests.
+
+import ./make-test-python.nix ({ pkgs, ...} : {
+  name = "lxd-nftables";
+  meta = with pkgs.stdenv.lib.maintainers; {
+    maintainers = [ patryk27 ];
+  };
+
+  machine = { lib, ... }: {
+    virtualisation = {
+      lxd.enable = true;
+    };
+
+    networking = {
+      firewall.enable = false;
+      nftables.enable = true;
+      nftables.ruleset = ''
+        table inet filter {
+          chain incoming {
+            type filter hook input priority 0;
+            policy accept;
+          }
+
+          chain forward {
+            type filter hook forward priority 0;
+            policy accept;
+          }
+
+          chain output {
+            type filter hook output priority 0;
+            policy accept;
+          }
+        }
+      '';
+    };
+  };
+
+  testScript = ''
+    machine.wait_for_unit("network.target")
+
+    with subtest("When nftables are enabled, lxd doesn't depend on iptables anymore"):
+        machine.succeed("lsmod | grep nf_tables")
+        machine.fail("lsmod | grep ip_tables")
+  '';
+})
diff --git a/nixos/tests/lxd.nix b/nixos/tests/lxd.nix
new file mode 100644
index 000000000000..db2d44dff557
--- /dev/null
+++ b/nixos/tests/lxd.nix
@@ -0,0 +1,135 @@
+import ./make-test-python.nix ({ pkgs, ...} :
+
+let
+  # Since we don't have access to the internet during the tests, we have to
+  # pre-fetch lxd containers beforehand.
+  #
+  # I've chosen to import Alpine Linux, because its image is turbo-tiny and,
+  # generally, sufficient for our tests.
+
+  alpine-meta = pkgs.fetchurl {
+    url = "https://uk.images.linuxcontainers.org/images/alpine/3.11/i386/default/20200608_13:00/lxd.tar.xz";
+    sha256 = "1hkvaj3rr333zmx1759njy435lps33gl4ks8zfm7m4nqvipm26a0";
+  };
+
+  alpine-rootfs = pkgs.fetchurl {
+    url = "https://uk.images.linuxcontainers.org/images/alpine/3.11/i386/default/20200608_13:00/rootfs.tar.xz";
+    sha256 = "1v82zdra4j5xwsff09qlp7h5vbsg54s0j7rdg4rynichfid3r347";
+  };
+
+  lxd-config = pkgs.writeText "config.yaml" ''
+    storage_pools:
+      - name: default
+        driver: dir
+        config:
+          source: /var/lxd-pool
+
+    networks:
+      - name: lxdbr0
+        type: bridge
+        config:
+          ipv4.address: auto
+          ipv6.address: none
+
+    profiles:
+      - name: default
+        devices:
+          eth0:
+            name: eth0
+            network: lxdbr0
+            type: nic
+          root:
+            path: /
+            pool: default
+            type: disk
+  '';
+
+in {
+  name = "lxd";
+  meta = with pkgs.stdenv.lib.maintainers; {
+    maintainers = [ patryk27 ];
+  };
+
+  machine = { lib, ... }: {
+    virtualisation = {
+      # Since we're testing `limits.cpu`, we've gotta have a known number of
+      # cores to lay on
+      cores = 2;
+
+      # Ditto, for `limits.memory`
+      memorySize = 512;
+
+      lxc.lxcfs.enable = true;
+      lxd.enable = true;
+    };
+  };
+
+  testScript = ''
+    machine.wait_for_unit("sockets.target")
+    machine.wait_for_unit("lxd.service")
+
+    # It takes additional second for lxd to settle
+    machine.sleep(1)
+
+    # lxd expects the pool's directory to already exist
+    machine.succeed("mkdir /var/lxd-pool")
+
+    machine.succeed(
+        "cat ${lxd-config} | lxd init --preseed"
+    )
+
+    machine.succeed(
+        "lxc image import ${alpine-meta} ${alpine-rootfs} --alias alpine"
+    )
+
+    with subtest("Containers can be launched and destroyed"):
+        machine.succeed("lxc launch alpine test")
+        machine.succeed("lxc exec test true")
+        machine.succeed("lxc delete -f test")
+
+    with subtest("Containers are being mounted with lxcfs inside"):
+        machine.succeed("lxc launch alpine test")
+
+        ## ---------- ##
+        ## limits.cpu ##
+
+        machine.succeed("lxc config set test limits.cpu 1")
+
+        # Since Alpine doesn't have `nproc` pre-installed, we've gotta resort
+        # to the primal methods
+        assert (
+            "1"
+            == machine.succeed("lxc exec test grep -- -c ^processor /proc/cpuinfo").strip()
+        )
+
+        machine.succeed("lxc config set test limits.cpu 2")
+
+        assert (
+            "2"
+            == machine.succeed("lxc exec test grep -- -c ^processor /proc/cpuinfo").strip()
+        )
+
+        ## ------------- ##
+        ## limits.memory ##
+
+        machine.succeed("lxc config set test limits.memory 64MB")
+
+        assert (
+            "MemTotal:          62500 kB"
+            == machine.succeed("lxc exec test grep -- MemTotal /proc/meminfo").strip()
+        )
+
+        machine.succeed("lxc config set test limits.memory 128MB")
+
+        assert (
+            "MemTotal:         125000 kB"
+            == machine.succeed("lxc exec test grep -- MemTotal /proc/meminfo").strip()
+        )
+
+        machine.succeed("lxc delete -f test")
+
+    with subtest("Unless explicitly changed, lxd leans on iptables"):
+        machine.succeed("lsmod | grep ip_tables")
+        machine.fail("lsmod | grep nf_tables")
+  '';
+})