Merge master into staging-next

14 files changed, 329 insertions, 19 deletions

diff --git a/nixos/doc/manual/administration/imperative-containers.section.md b/nixos/doc/manual/administration/imperative-containers.section.md
index f45991780c4b..852305ad8148 100644
--- a/nixos/doc/manual/administration/imperative-containers.section.md
+++ b/nixos/doc/manual/administration/imperative-containers.section.md
@@ -77,7 +77,7 @@ Linux foo 3.4.82 #1-NixOS SMP Thu Mar 20 14:44:05 UTC 2014 x86_64 GNU/Linux
 
 There are several ways to change the configuration of the container.
 First, on the host, you can edit
-`/var/lib/container/name/etc/nixos/configuration.nix`, and run
+`/var/lib/nixos-containers/foo/etc/nixos/configuration.nix`, and run
 
 ```ShellSession
 # nixos-container update foo
diff --git a/nixos/doc/manual/development/unit-handling.section.md b/nixos/doc/manual/development/unit-handling.section.md
index 32d44dbfff05..d5ba6a9529d0 100644
--- a/nixos/doc/manual/development/unit-handling.section.md
+++ b/nixos/doc/manual/development/unit-handling.section.md
@@ -63,3 +63,42 @@ checks:
     is **restart**ed with the others. If it is set, both the service and the
     socket are **stop**ped and the socket is **start**ed, leaving socket
     activation to start the service when it's needed.
+
+## Sysinit reactivation {#sec-sysinit-reactivation}
+
+[`sysinit.target`](https://www.freedesktop.org/software/systemd/man/latest/systemd.special.html#sysinit.target)
+is a systemd target that encodes system initialization (i.e. early startup). A
+few units that need to run very early in the bootup process are ordered to
+finish before this target is reached. Probably the most notable one of these is
+`systemd-tmpfiles-setup.service`. We will refer to these units as "sysinit
+units".
+
+"Normal" systemd units, by default, are ordered AFTER `sysinit.target`. In
+other words, these "normal" units expect all services ordered before
+`sysinit.target` to have finished without explicity declaring this dependency
+relationship for each dependency. See the [systemd
+bootup](https://www.freedesktop.org/software/systemd/man/latest/bootup.html)
+for more details on the bootup process.
+
+When restarting both a unit ordered before `sysinit.target` as well as one
+after, this presents a problem because they would be started at the same time
+as they do not explicitly declare their dependency relations.
+
+To solve this, NixOS has an artificial `sysinit-reactivation.target` which
+allows you to ensure that services ordered before `sysinit.target` are
+restarted correctly. This applies both to the ordering between these sysinit
+services as well as ensuring that sysinit units are restarted before "normal"
+units.
+
+To make an existing sysinit service restart correctly during system switch, you
+have to declare:
+
+```nix
+systemd.services.my-sysinit = {
+  requiredBy = [ "sysinit-reactivation.target" ];
+  before = [ "sysinit-reactivation.target" ];
+  restartTriggers = [ config.environment.etc."my-sysinit.d".source ];
+};
+```
+
+You need to configure appropriate `restartTriggers` specific to your service.
diff --git a/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md b/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md
index ccadb819e061..5d17a9c98514 100644
--- a/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md
+++ b/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md
@@ -37,7 +37,7 @@ of actions is always the same:
 - Forget about the failed state of units (`systemctl reset-failed`)
 - Reload systemd (`systemctl daemon-reload`)
 - Reload systemd user instances (`systemctl --user daemon-reload`)
-- Set up tmpfiles (`systemd-tmpfiles --create`)
+- Reactivate sysinit (`systemctl restart sysinit-reactivation.target`)
 - Reload units (`systemctl reload`)
 - Restart units (`systemctl restart`)
 - Start units (`systemctl start`)
diff --git a/nixos/doc/manual/release-notes/rl-2405.section.md b/nixos/doc/manual/release-notes/rl-2405.section.md
index 1d402a51efb5..f052cde442ce 100644
--- a/nixos/doc/manual/release-notes/rl-2405.section.md
+++ b/nixos/doc/manual/release-notes/rl-2405.section.md
@@ -116,6 +116,11 @@ The pre-existing [services.ankisyncd](#opt-services.ankisyncd.enable) has been m
 
 - The executable file names for `firefox-devedition`, `firefox-beta`, `firefox-esr` now matches their package names, which is consistent with the `firefox-*-bin` packages. The desktop entries are also updated so that you can have multiple editions of firefox in your app launcher.
 
+- switch-to-configuration does not directly call systemd-tmpfiles anymore.
+  Instead, the new artificial sysinit-reactivation.target is introduced which
+  allows to restart multiple services that are ordered before sysinit.target
+  and respect the ordering between the services.
+
 - The `systemd.oomd` module behavior is changed as:
 
   - Raise ManagedOOMMemoryPressureLimit from 50% to 80%. This should make systemd-oomd kill things less often, and fix issues like [this](https://pagure.io/fedora-workstation/issue/358).
diff --git a/nixos/maintainers/option-usages.nix b/nixos/maintainers/option-usages.nix
index 11247666ecda..e9bafa21a58a 100644
--- a/nixos/maintainers/option-usages.nix
+++ b/nixos/maintainers/option-usages.nix
@@ -9,17 +9,17 @@
 
 # This file is made to be used as follow:
 #
-#   $ nix-instantiate ./option-usage.nix --argstr testOption service.xserver.enable -A txtContent --eval
+#   $ nix-instantiate ./option-usages.nix --argstr testOption service.xserver.enable -A txtContent --eval
 #
 # or
 #
-#   $ nix-build ./option-usage.nix --argstr testOption service.xserver.enable -A txt -o service.xserver.enable._txt
+#   $ nix-build ./option-usages.nix --argstr testOption service.xserver.enable -A txt -o service.xserver.enable._txt
 #
 # Other targets exists such as `dotContent`, `dot`, and `pdf`.  If you are
 # looking for the option usage of multiple options, you can provide a list
 # as argument.
 #
-#   $ nix-build ./option-usage.nix --arg testOptions \
+#   $ nix-build ./option-usages.nix --arg testOptions \
 #      '["boot.loader.gummiboot.enable" "boot.loader.gummiboot.timeout"]' \
 #      -A txt -o gummiboot.list
 #
diff --git a/nixos/modules/system/activation/switch-to-configuration.pl b/nixos/modules/system/activation/switch-to-configuration.pl
index e2f66a287bc4..ba45231465fb 100755
--- a/nixos/modules/system/activation/switch-to-configuration.pl
+++ b/nixos/modules/system/activation/switch-to-configuration.pl
@@ -889,9 +889,15 @@ while (my $f = <$list_active_users>) {
 
 close($list_active_users) || die("Unable to close the file handle to loginctl");
 
-# Set the new tmpfiles
-print STDERR "setting up tmpfiles\n";
-system("$new_systemd/bin/systemd-tmpfiles", "--create", "--remove", "--exclude-prefix=/dev") == 0 or $res = 3;
+# Restart sysinit-reactivation.target.
+# This target only exists to restart services ordered before sysinit.target. We
+# cannot use X-StopOnReconfiguration to restart sysinit.target because then ALL
+# services of the system would be restarted since all normal services have a
+# default dependency on sysinit.target. sysinit-reactivation.target ensures
+# that services ordered BEFORE sysinit.target get re-started in the correct
+# order. Ordering between these services is respected.
+print STDERR "restarting sysinit-reactivation.target\n";
+system("$new_systemd/bin/systemctl", "restart", "sysinit-reactivation.target") == 0 or $res = 4;
 
 # Before reloading we need to ensure that the units are still active. They may have been
 # deactivated because one of their requirements got stopped. If they are inactive
diff --git a/nixos/modules/system/boot/systemd.nix b/nixos/modules/system/boot/systemd.nix
index c3902007906a..46c3f66f02dc 100644
--- a/nixos/modules/system/boot/systemd.nix
+++ b/nixos/modules/system/boot/systemd.nix
@@ -569,6 +569,13 @@ in
         unitConfig.X-StopOnReconfiguration = true;
       };
 
+    # This target only exists so that services ordered before sysinit.target
+    # are restarted in the correct order, notably BEFORE the other services,
+    # when switching configurations.
+    systemd.targets.sysinit-reactivation = {
+      description = "Reactivate sysinit units";
+    };
+
     systemd.units =
          mapAttrs' (n: v: nameValuePair "${n}.path"    (pathToUnit    n v)) cfg.paths
       // mapAttrs' (n: v: nameValuePair "${n}.service" (serviceToUnit n v)) cfg.services
diff --git a/nixos/modules/system/boot/systemd/tmpfiles.nix b/nixos/modules/system/boot/systemd/tmpfiles.nix
index 183e2033ecb0..dae23eddd1e2 100644
--- a/nixos/modules/system/boot/systemd/tmpfiles.nix
+++ b/nixos/modules/system/boot/systemd/tmpfiles.nix
@@ -150,6 +150,41 @@ in
       "systemd-tmpfiles-setup.service"
     ];
 
+    # Allow systemd-tmpfiles to be restarted by switch-to-configuration. This
+    # service is not pulled into the normal boot process. It only exists for
+    # switch-to-configuration.
+    #
+    # This needs to be a separate unit because it does not execute
+    # systemd-tmpfiles with `--boot` as that is supposed to only be executed
+    # once at boot time.
+    #
+    # Keep this aligned with the upstream `systemd-tmpfiles-setup.service` unit.
+    systemd.services."systemd-tmpfiles-resetup" = {
+      description = "Re-setup tmpfiles on a system that is already running.";
+
+      requiredBy = [ "sysinit-reactivation.target" ];
+      after = [ "local-fs.target" "systemd-sysusers.service" "systemd-journald.service" ];
+      before = [ "sysinit-reactivation.target" "shutdown.target" ];
+      conflicts = [ "shutdown.target" ];
+      restartTriggers = [ config.environment.etc."tmpfiles.d".source ];
+
+      unitConfig.DefaultDependencies = false;
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = "systemd-tmpfiles --create --remove --exclude-prefix=/dev";
+        SuccessExitStatus = "DATAERR CANTCREAT";
+        ImportCredential = [
+          "tmpfiles.*"
+          "loging.motd"
+          "login.issue"
+          "network.hosts"
+          "ssh.authorized_keys.root"
+        ];
+      };
+    };
+
     environment.etc = {
       "tmpfiles.d".source = (pkgs.symlinkJoin {
         name = "tmpfiles.d";
diff --git a/nixos/modules/virtualisation/incus.nix b/nixos/modules/virtualisation/incus.nix
index 3e48f8873ed4..ea4cb916aa08 100644
--- a/nixos/modules/virtualisation/incus.nix
+++ b/nixos/modules/virtualisation/incus.nix
@@ -150,10 +150,12 @@ in
       after = [
         "network-online.target"
         "lxcfs.service"
-      ] ++ (lib.optional cfg.socketActivation "incus.socket");
+        "incus.socket"
+      ];
       requires = [
         "lxcfs.service"
-      ] ++ (lib.optional cfg.socketActivation "incus.socket");
+        "incus.socket"
+      ];
       wants = [
         "network-online.target"
       ];
@@ -183,7 +185,7 @@ in
       };
     };
 
-    systemd.sockets.incus = lib.mkIf cfg.socketActivation {
+    systemd.sockets.incus = {
       description = "Incus UNIX socket";
       wantedBy = [ "sockets.target" ];
 
@@ -191,7 +193,6 @@ in
         ListenStream = "/var/lib/incus/unix.socket";
         SocketMode = "0660";
         SocketGroup = "incus-admin";
-        Service = "incus.service";
       };
     };
 
diff --git a/nixos/modules/virtualisation/lxd.nix b/nixos/modules/virtualisation/lxd.nix
index 885fb4e07853..e0d61b175494 100644
--- a/nixos/modules/virtualisation/lxd.nix
+++ b/nixos/modules/virtualisation/lxd.nix
@@ -214,16 +214,14 @@ in {
         LimitNPROC = "infinity";
         TasksMax = "infinity";
 
-        Restart = "on-failure";
-        TimeoutStartSec = "${cfg.startTimeout}s";
-        TimeoutStopSec = "30s";
-
         # By default, `lxd` loads configuration files from hard-coded
         # `/usr/share/lxc/config` - since this is a no-go for us, we have to
         # explicitly tell it where the actual configuration files are
         Environment = lib.mkIf (config.virtualisation.lxc.lxcfs.enable)
           "LXD_LXC_TEMPLATE_CONFIG=${pkgs.lxcfs}/share/lxc/config";
       };
+
+      unitConfig.ConditionPathExists = "!/var/lib/incus/.migrated-from-lxd";
     };
 
     systemd.services.lxd-preseed = lib.mkIf (cfg.preseed != null) {
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index cf59c933e9ab..a3b8ba3277d0 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -820,6 +820,7 @@ in {
   syncthing-init = handleTest ./syncthing-init.nix {};
   syncthing-many-devices = handleTest ./syncthing-many-devices.nix {};
   syncthing-relay = handleTest ./syncthing-relay.nix {};
+  sysinit-reactivation = runTest ./sysinit-reactivation.nix;
   systemd = handleTest ./systemd.nix {};
   systemd-analyze = handleTest ./systemd-analyze.nix {};
   systemd-binfmt = handleTestOn ["x86_64-linux"] ./systemd-binfmt.nix {};
diff --git a/nixos/tests/incus/default.nix b/nixos/tests/incus/default.nix
index c88974605e30..26e8a4ac4c77 100644
--- a/nixos/tests/incus/default.nix
+++ b/nixos/tests/incus/default.nix
@@ -6,9 +6,8 @@
 }:
 {
   container = import ./container.nix { inherit system pkgs; };
+  lxd-to-incus = import ./lxd-to-incus.nix { inherit system pkgs; };
   preseed = import ./preseed.nix { inherit system pkgs; };
   socket-activated = import ./socket-activated.nix { inherit system pkgs; };
-  virtual-machine = handleTestOn [ "x86_64-linux" ] ./virtual-machine.nix {
-    inherit system pkgs;
-  };
+  virtual-machine = handleTestOn [ "x86_64-linux" ] ./virtual-machine.nix { inherit system pkgs; };
 }
diff --git a/nixos/tests/incus/lxd-to-incus.nix b/nixos/tests/incus/lxd-to-incus.nix
new file mode 100644
index 000000000000..67245b54e752
--- /dev/null
+++ b/nixos/tests/incus/lxd-to-incus.nix
@@ -0,0 +1,112 @@
+import ../make-test-python.nix (
+
+  { pkgs, lib, ... }:
+
+  let
+    releases = import ../../release.nix { configuration.documentation.enable = lib.mkForce false; };
+
+    container-image-metadata = releases.lxdContainerMeta.${pkgs.stdenv.hostPlatform.system};
+    container-image-rootfs = releases.lxdContainerImage.${pkgs.stdenv.hostPlatform.system};
+  in
+  {
+    name = "lxd-to-incus";
+
+    meta = {
+      maintainers = lib.teams.lxc.members;
+    };
+
+    nodes.machine =
+      { lib, ... }:
+      {
+        environment.systemPackages = [ pkgs.lxd-to-incus ];
+
+        virtualisation = {
+          diskSize = 6144;
+          cores = 2;
+          memorySize = 2048;
+
+          lxd.enable = true;
+          lxd.preseed = {
+            networks = [
+              {
+                name = "nixostestbr0";
+                type = "bridge";
+                config = {
+                  "ipv4.address" = "10.0.100.1/24";
+                  "ipv4.nat" = "true";
+                };
+              }
+            ];
+            profiles = [
+              {
+                name = "default";
+                devices = {
+                  eth0 = {
+                    name = "eth0";
+                    network = "nixostestbr0";
+                    type = "nic";
+                  };
+                  root = {
+                    path = "/";
+                    pool = "nixostest_pool";
+                    size = "35GiB";
+                    type = "disk";
+                  };
+                };
+              }
+              {
+                name = "nixos_notdefault";
+                devices = { };
+              }
+            ];
+            storage_pools = [
+              {
+                name = "nixostest_pool";
+                driver = "dir";
+              }
+            ];
+          };
+
+          incus.enable = true;
+        };
+      };
+
+    testScript = ''
+      def lxd_wait_for_preseed(_) -> bool:
+        _, output = machine.systemctl("is-active lxd-preseed.service")
+        return ("inactive" in output)
+
+      def lxd_instance_is_up(_) -> bool:
+          status, _ = machine.execute("lxc exec container --disable-stdin --force-interactive /run/current-system/sw/bin/true")
+          return status == 0
+
+      def incus_instance_is_up(_) -> bool:
+          status, _ = machine.execute("incus exec container --disable-stdin --force-interactive /run/current-system/sw/bin/true")
+          return status == 0
+
+      with machine.nested("initialize lxd and resources"):
+        machine.wait_for_unit("sockets.target")
+        machine.wait_for_unit("lxd.service")
+        retry(lxd_wait_for_preseed)
+
+        machine.succeed("lxc image import ${container-image-metadata}/*/*.tar.xz ${container-image-rootfs}/*/*.tar.xz --alias nixos")
+        machine.succeed("lxc launch nixos container")
+        retry(lxd_instance_is_up)
+
+      machine.wait_for_unit("incus.service")
+
+      with machine.nested("run migration"):
+          machine.succeed("lxd-to-incus --yes")
+
+      with machine.nested("verify resources migrated to incus"):
+          machine.succeed("incus config show container")
+          retry(incus_instance_is_up)
+          machine.succeed("incus exec container -- true")
+          machine.succeed("incus profile show default | grep nixostestbr0")
+          machine.succeed("incus profile show default | grep nixostest_pool")
+          machine.succeed("incus profile show nixos_notdefault")
+          machine.succeed("incus storage show nixostest_pool")
+          machine.succeed("incus network show nixostestbr0")
+    '';
+  }
+)
diff --git a/nixos/tests/sysinit-reactivation.nix b/nixos/tests/sysinit-reactivation.nix
new file mode 100644
index 000000000000..1a0caecb610a
--- /dev/null
+++ b/nixos/tests/sysinit-reactivation.nix
@@ -0,0 +1,107 @@
+# This runs to two scenarios but in one tests:
+# - A post-sysinit service needs to be restarted AFTER tmpfiles was restarted.
+# - A service needs to be restarted BEFORE tmpfiles is restarted
+
+{ lib, ... }:
+
+let
+  makeGeneration = generation: {
+    "${generation}".configuration = {
+      systemd.services.pre-sysinit-before-tmpfiles.environment.USER =
+        lib.mkForce "${generation}-tmpfiles-user";
+
+      systemd.services.pre-sysinit-after-tmpfiles.environment = {
+        NEEDED_PATH = lib.mkForce "/run/${generation}-needed-by-pre-sysinit-after-tmpfiles";
+        PATH_TO_CREATE = lib.mkForce "/run/${generation}-needed-by-post-sysinit";
+      };
+
+      systemd.services.post-sysinit.environment = {
+        NEEDED_PATH = lib.mkForce "/run/${generation}-needed-by-post-sysinit";
+        PATH_TO_CREATE = lib.mkForce "/run/${generation}-created-by-post-sysinit";
+      };
+
+      systemd.tmpfiles.settings.test = lib.mkForce {
+        "/run/${generation}-needed-by-pre-sysinit-after-tmpfiles".f.user =
+          "${generation}-tmpfiles-user";
+      };
+    };
+  };
+in
+
+{
+
+  name = "sysinit-reactivation";
+
+  meta.maintainers = with lib.maintainers; [ nikstur ];
+
+  nodes.machine = { config, lib, pkgs, ... }: {
+    systemd.services.pre-sysinit-before-tmpfiles = {
+      wantedBy = [ "sysinit.target" ];
+      requiredBy = [ "sysinit-reactivation.target" ];
+      before = [ "systemd-tmpfiles-setup.service" "systemd-tmpfiles-resetup.service" ];
+      unitConfig.DefaultDependencies = false;
+      serviceConfig.Type = "oneshot";
+      serviceConfig.RemainAfterExit = true;
+      environment.USER = "tmpfiles-user";
+      script = "${pkgs.shadow}/bin/useradd $USER";
+    };
+
+    systemd.services.pre-sysinit-after-tmpfiles = {
+      wantedBy = [ "sysinit.target" ];
+      requiredBy = [ "sysinit-reactivation.target" ];
+      after = [ "systemd-tmpfiles-setup.service" "systemd-tmpfiles-resetup.service" ];
+      unitConfig.DefaultDependencies = false;
+      serviceConfig.Type = "oneshot";
+      serviceConfig.RemainAfterExit = true;
+      environment = {
+        NEEDED_PATH = "/run/needed-by-pre-sysinit-after-tmpfiles";
+        PATH_TO_CREATE = "/run/needed-by-post-sysinit";
+      };
+      script = ''
+        if [[ -e $NEEDED_PATH ]]; then
+          touch $PATH_TO_CREATE
+        fi
+      '';
+    };
+
+    systemd.services.post-sysinit = {
+      wantedBy = [ "default.target" ];
+      serviceConfig.Type = "oneshot";
+      serviceConfig.RemainAfterExit = true;
+      environment = {
+        NEEDED_PATH = "/run/needed-by-post-sysinit";
+        PATH_TO_CREATE = "/run/created-by-post-sysinit";
+      };
+      script = ''
+        if [[ -e $NEEDED_PATH ]]; then
+          touch $PATH_TO_CREATE
+        fi
+      '';
+    };
+
+    systemd.tmpfiles.settings.test = {
+      "/run/needed-by-pre-sysinit-after-tmpfiles".f.user =
+        "tmpfiles-user";
+    };
+
+    specialisation = lib.mkMerge [
+      (makeGeneration "second")
+      (makeGeneration "third")
+    ];
+  };
+
+  testScript = { nodes, ... }: ''
+    def switch(generation):
+      toplevel = "${nodes.machine.system.build.toplevel}";
+      machine.succeed(f"{toplevel}/specialisation/{generation}/bin/switch-to-configuration switch")
+
+    machine.wait_for_unit("default.target")
+    machine.succeed("test -e /run/created-by-post-sysinit")
+
+    switch("second")
+    machine.succeed("test -e /run/second-created-by-post-sysinit")
+
+    switch("third")
+    machine.succeed("test -e /run/third-created-by-post-sysinit")
+  '';
+}