Merge pull request #264929 from adamcstephens/incus/generator

5 files changed, 187 insertions, 68 deletions

diff --git a/nixos/modules/virtualisation/lxc-container.nix b/nixos/modules/virtualisation/lxc-container.nix
index 61d7c4cb73fe..4db4df02fe8c 100644
--- a/nixos/modules/virtualisation/lxc-container.nix
+++ b/nixos/modules/virtualisation/lxc-container.nix
@@ -1,26 +1,16 @@
 { lib, config, pkgs, ... }:
 
-let
-  cfg = config.virtualisation.lxc;
-in {
+{
+  meta.maintainers = with lib.maintainers; [ adamcstephens ];
+
   imports = [
     ./lxc-instance-common.nix
+
+    (lib.mkRemovedOptionModule [ "virtualisation" "lxc" "nestedContainer" ] "")
+    (lib.mkRemovedOptionModule [ "virtualisation" "lxc" "privilegedContainer" ] "")
   ];
 
-  options = {
-    virtualisation.lxc = {
-      nestedContainer = lib.mkEnableOption (lib.mdDoc ''
-        Whether this container is configured as a nested container. On LXD containers this is recommended
-        for all containers and is enabled with `security.nesting = true`.
-      '');
-
-      privilegedContainer = lib.mkEnableOption (lib.mdDoc ''
-        Whether this LXC container will be running as a privileged container or not. If set to `true` then
-        additional configuration will be applied to the `systemd` instance running within the container as
-        recommended by [distrobuilder](https://linuxcontainers.org/distrobuilder/introduction/).
-      '');
-    };
-  };
+  options = { };
 
   config = {
     boot.isContainer = true;
@@ -85,34 +75,10 @@ in {
       ${pkgs.coreutils}/bin/ln -fs "$1/init" /sbin/init
     '';
 
-    systemd.additionalUpstreamSystemUnits = lib.mkIf cfg.nestedContainer ["systemd-udev-trigger.service"];
-
-    # Add the overrides from lxd distrobuilder
-    # https://github.com/lxc/distrobuilder/blob/05978d0d5a72718154f1525c7d043e090ba7c3e0/distrobuilder/main.go#L630
-    systemd.packages = [
-      (pkgs.writeTextFile {
-        name = "systemd-lxc-service-overrides";
-        destination = "/etc/systemd/system/service.d/zzz-lxc-service.conf";
-        text = ''
-          [Service]
-          ProcSubset=all
-          ProtectProc=default
-          ProtectControlGroups=no
-          ProtectKernelTunables=no
-          NoNewPrivileges=no
-          LoadCredential=
-        '' + lib.optionalString cfg.privilegedContainer ''
-          # Additional settings for privileged containers
-          ProtectHome=no
-          ProtectSystem=no
-          PrivateDevices=no
-          PrivateTmp=no
-          ProtectKernelLogs=no
-          ProtectKernelModules=no
-          ReadWritePaths=
-        '';
-      })
-    ];
+    # networkd depends on this, but systemd module disables this for containers
+    systemd.additionalUpstreamSystemUnits = ["systemd-udev-trigger.service"];
+
+    systemd.packages = [ pkgs.distrobuilder.generator ];
 
     system.activationScripts.installInitScript = lib.mkForce ''
       ln -fs $systemConfig/init /sbin/init
diff --git a/nixos/tests/incus/container.nix b/nixos/tests/incus/container.nix
index 79b9e2fbabdc..49a22c08aad1 100644
--- a/nixos/tests/incus/container.nix
+++ b/nixos/tests/incus/container.nix
@@ -73,5 +73,33 @@ in
         meminfo = machine.succeed("incus exec container grep -- MemTotal /proc/meminfo").strip()
         meminfo_bytes = " ".join(meminfo.split(' ')[-2:])
         assert meminfo_bytes == "125000 kB", f"Wrong amount of memory reported from /proc/meminfo, want: '125000 kB', got: '{meminfo_bytes}'"
+
+    with subtest("lxc-container generator configures plain container"):
+        machine.execute("incus delete --force container")
+        machine.succeed("incus launch nixos container")
+        with machine.nested("Waiting for instance to start and be usable"):
+          retry(instance_is_up)
+
+        machine.succeed("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
+
+    with subtest("lxc-container generator configures nested container"):
+        machine.execute("incus delete --force container")
+        machine.succeed("incus launch nixos container --config security.nesting=true")
+        with machine.nested("Waiting for instance to start and be usable"):
+          retry(instance_is_up)
+
+        machine.fail("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
+        target = machine.succeed("incus exec container readlink -- -f /run/systemd/system/systemd-binfmt.service").strip()
+        assert target == "/dev/null", "lxc generator did not correctly mask /run/systemd/system/systemd-binfmt.service"
+
+    with subtest("lxc-container generator configures privileged container"):
+        machine.execute("incus delete --force container")
+        machine.succeed("incus launch nixos container --config security.privileged=true")
+        with machine.nested("Waiting for instance to start and be usable"):
+          retry(instance_is_up)
+        # give generator an extra second to run
+        machine.sleep(1)
+
+        machine.succeed("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
   '';
 })
diff --git a/pkgs/tools/virtualization/distrobuilder/default.nix b/pkgs/tools/virtualization/distrobuilder/default.nix
index a1a0cfde0fe7..e6b867139165 100644
--- a/pkgs/tools/virtualization/distrobuilder/default.nix
+++ b/pkgs/tools/virtualization/distrobuilder/default.nix
@@ -8,7 +8,8 @@
 , gnutar
 , squashfsTools
 , debootstrap
-, fetchpatch
+, callPackage
+, nixosTests
 }:
 
 let
@@ -22,34 +23,20 @@ let
 in
 buildGoModule rec {
   pname = "distrobuilder";
-  version = "2.1";
+  version = "3.0";
 
-  vendorHash = "sha256-yRMsf8KfpNmVUX4Rn4ZPLUPFZCT/g78MKAfgbFDPVkE=";
+  vendorHash = "sha256-pFrEkZnrcx0d3oM1klQrNHH+MiLvO4V1uFQdE0kXUqM=";
 
   src = fetchFromGitHub {
     owner = "lxc";
     repo = "distrobuilder";
-    rev = "distrobuilder-${version}";
-    sha256 = "sha256-t3ECLtb0tvIzTWgjmVQDFza+kcm3abTZZMSGYjvw1qQ=";
+    rev = "refs/tags/distrobuilder-${version}";
+    sha256 = "sha256-JfME9VaqaQnrhnzhSLGUy9uU+tki1hXdnwqBUD/5XH0=";
     fetchSubmodules = false;
   };
 
   buildInputs = bins;
 
-  patches = [
-    # go.mod update: needed to to include a newer lxd which contains
-    # https://github.com/canonical/lxd/commit/d83f061a21f509d42b7a334b97403d2a019a7b52
-    # which is needed to fix the build w/glibc-2.36.
-    (fetchpatch {
-      url = "https://github.com/lxc/distrobuilder/commit/5346bcc77dd7f141a36a8da851f016d0b929835e.patch";
-      sha256 = "sha256-H6cSbY0v/FThx72AvoAvUCs2VCYN/PQ0W4H82mQQ3SI=";
-    })
-    # Fixup to keep it building after go.mod update.
-    (fetchpatch {
-      url = "https://github.com/lxc/distrobuilder/commit/2c8cbfbf603e7446efce9f30812812336ccf4f2c.patch";
-      sha256 = "sha256-qqofghcHGosR2qycGb02c8rwErFyRRhsRKdQfyah8Ds=";
-    })
-  ];
 
   # tests require a local keyserver (mkg20001/nixpkgs branch distrobuilder-with-tests) but gpg is currently broken in tests
   doCheck = false;
@@ -63,12 +50,18 @@ buildGoModule rec {
     wrapProgram $out/bin/distrobuilder --prefix PATH ":" ${lib.makeBinPath bins}
   '';
 
-  meta = with lib; {
+  passthru = {
+    tests.incus = nixosTests.incus.container;
+
+    generator = callPackage ./generator.nix { inherit src version; };
+  };
+
+  meta = {
     description = "System container image builder for LXC and LXD";
     homepage = "https://github.com/lxc/distrobuilder";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ megheaiulian ];
-    platforms = platforms.linux;
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ megheaiulian adamcstephens ];
+    platforms = lib.platforms.linux;
     mainProgram = "distrobuilder";
   };
 }
diff --git a/pkgs/tools/virtualization/distrobuilder/generator.nix b/pkgs/tools/virtualization/distrobuilder/generator.nix
new file mode 100644
index 000000000000..e514a7df2e08
--- /dev/null
+++ b/pkgs/tools/virtualization/distrobuilder/generator.nix
@@ -0,0 +1,19 @@
+{ stdenvNoCC, lib, src, version, makeWrapper, coreutils, findutils, gnugrep, systemd }:
+
+stdenvNoCC.mkDerivation {
+  name = "distrobuilder-nixos-generator";
+
+  inherit src version;
+
+  patches = [
+    ./nixos-generator.patch
+  ];
+
+  dontBuild = true;
+  nativeBuildInputs = [ makeWrapper ];
+
+  installPhase = ''
+    install -D -m 0555 distrobuilder/lxc.generator $out/lib/systemd/system-generators/lxc
+    wrapProgram $out/lib/systemd/system-generators/lxc --prefix PATH : ${lib.makeBinPath [coreutils findutils gnugrep systemd]}:${systemd}/lib/systemd
+  '';
+}
diff --git a/pkgs/tools/virtualization/distrobuilder/nixos-generator.patch b/pkgs/tools/virtualization/distrobuilder/nixos-generator.patch
new file mode 100644
index 000000000000..6194f33e1918
--- /dev/null
+++ b/pkgs/tools/virtualization/distrobuilder/nixos-generator.patch
@@ -0,0 +1,113 @@
+diff --git a/distrobuilder/lxc.generator b/distrobuilder/lxc.generator
+index 0ad81d1..69dbfe7 100644
+--- a/distrobuilder/lxc.generator
++++ b/distrobuilder/lxc.generator
+@@ -25,16 +25,6 @@ is_incus_vm() {
+ 	[ -e /dev/virtio-ports/org.linuxcontainers.incus ]
+ }
+ 
+-# is_in_path succeeds if the given file exists in on of the paths
+-is_in_path() {
+-	# Don't use $PATH as that may not include all relevant paths
+-	for path in /bin /sbin /usr/bin /usr/sbin /usr/local/bin /usr/local/sbin; do
+-		[ -e "${path}/$1" ] && return 0
+-	done
+-
+-	return 1
+-}
+-
+ ## Fix functions
+ # fix_ro_paths avoids udevd issues with /sys and /proc being writable
+ fix_ro_paths() {
+@@ -45,35 +35,6 @@ BindReadOnlyPaths=/sys /proc
+ EOF
+ }
+ 
+-# fix_nm_link_state forces the network interface to a DOWN state ahead of NetworkManager starting up
+-fix_nm_link_state() {
+-	[ -e "/sys/class/net/$1" ] || return 0
+-	ip_path=
+-	if [ -f /sbin/ip ]; then
+-		ip_path=/sbin/ip
+-	elif [ -f /bin/ip ]; then
+-		ip_path=/bin/ip
+-	else
+-		return 0
+-	fi
+-	cat <<-EOF > /run/systemd/system/network-device-down.service
+-[Unit]
+-Description=Turn off network device
+-Before=NetworkManager.service
+-Before=systemd-networkd.service
+-[Service]
+-# do not turn off if there is a default route to 169.254.0.1, i.e. the device is a routed nic
+-ExecCondition=/bin/sh -c '! /usr/bin/grep -qs 00000000.0100FEA9 /proc/net/route'
+-ExecStart=-${ip_path} link set $1 down
+-Type=oneshot
+-RemainAfterExit=true
+-[Install]
+-WantedBy=default.target
+-EOF
+-	mkdir -p /run/systemd/system/default.target.wants
+-	ln -sf /run/systemd/system/network-device-down.service /run/systemd/system/default.target.wants/network-device-down.service
+-}
+-
+ # fix_systemd_override_unit generates a unit specific override
+ fix_systemd_override_unit() {
+ 	dropin_dir="/run/systemd/${1}.d"
+@@ -112,16 +73,7 @@ fix_systemd_mask() {
+ # fix_systemd_udev_trigger overrides the systemd-udev-trigger.service to match the latest version
+ # of the file which uses "ExecStart=-" instead of "ExecStart=".
+ fix_systemd_udev_trigger() {
+-	cmd=
+-	if [ -f /usr/bin/udevadm ]; then
+-		cmd=/usr/bin/udevadm
+-	elif [ -f /sbin/udevadm ]; then
+-		cmd=/sbin/udevadm
+-	elif [ -f /bin/udevadm ]; then
+-		cmd=/bin/udevadm
+-	else
+-		return 0
+-	fi
++	cmd=udevadm
+ 
+ 	mkdir -p /run/systemd/system/systemd-udev-trigger.service.d
+ 	cat <<-EOF > /run/systemd/system/systemd-udev-trigger.service.d/zzz-lxc-override.conf
+@@ -145,24 +97,12 @@ EOF
+ }
+ 
+ ## Main logic
+-# Nothing to do in Incus VM but deployed in case it is later converted to a container
+-is_incus_vm || is_lxd_vm && exit 0
+ 
+ # Exit immediately if not an Incus/LXC container
+ is_lxc_container || exit 0
+ 
+-# Check for NetworkManager
+-nm_exists=0
+-
+-is_in_path NetworkManager && nm_exists=1
+-
+ # Determine systemd version
+-for path in /usr/lib/systemd/systemd /lib/systemd/systemd; do
+-	[ -x "${path}" ] || continue
+-
+-	systemd_version="$("${path}" --version | head -n1 | cut -d' ' -f2)"
+-	break
+-done
++systemd_version="$(systemd --version | head -n1 | cut -d' ' -f2)"
+ 
+ # Determine distro name and release
+ ID=""
+@@ -222,11 +162,6 @@ ACTION=="add|change|move", ENV{ID_NET_DRIVER}=="veth", ENV{INTERFACE}=="eth[0-9]
+ EOF
+ fi
+ 
+-# Workarounds for NetworkManager in containers
+-if [ "${nm_exists}" -eq 1 ]; then
+-	fix_nm_link_state eth0
+-fi
+-
+ # Allow masking units created by the lxc system-generator.
+ for d in /etc/systemd/system /usr/lib/systemd/system /lib/systemd/system; do
+ 	if ! [ -d "${d}" ]; then