Merge staging-next into staging

9 files changed, 117 insertions, 80 deletions

diff --git a/nixos/doc/manual/development/developing-the-test-driver.chapter.md b/nixos/doc/manual/development/developing-the-test-driver.chapter.md
new file mode 100644
index 000000000000..4b70fe00af47
--- /dev/null
+++ b/nixos/doc/manual/development/developing-the-test-driver.chapter.md
@@ -0,0 +1,43 @@
+
+# Developing the NixOS Test Driver {#chap-developing-the-test-driver}
+
+The NixOS test framework is a project of its own.
+
+It consists of roughly the following components:
+
+ - `nixos/lib/test-driver`: The Python framework that sets up the test and runs the [`testScript`](#test-opt-testScript)
+ - `nixos/lib/testing`: The Nix code responsible for the wiring, written using the (NixOS) Module System.
+
+These components are exposed publicly through:
+
+ - `nixos/lib/default.nix`: The public interface that exposes the `nixos/lib/testing` entrypoint.
+ - `flake.nix`: Exposes the `lib.nixos`, including the public test interface.
+
+Beyond the test driver itself, its integration into NixOS and Nixpkgs is important.
+
+ - `pkgs/top-level/all-packages.nix`: Defines the `nixosTests` attribute, used
+   by the package `tests` attributes and OfBorg.
+ - `nixos/release.nix`: Defines the `tests` attribute built by Hydra, independently, but analogous to `nixosTests`
+ - `nixos/release-combined.nix`: Defines which tests are channel blockers.
+
+Finally, we have legacy entrypoints that users should move away from, but are cared for on a best effort basis.
+These include `pkgs.nixosTest`, `testing-python.nix` and `make-test-python.nix`.
+
+## Testing changes to the test framework {#sec-test-the-test-framework}
+
+When making significant changes to the test framework, we run the tests on Hydra, to avoid disrupting the larger NixOS project.
+
+For this, we use the `python-test-refactoring` branch in the `NixOS/nixpkgs` repository, and its [corresponding Hydra jobset](https://hydra.nixos.org/jobset/nixos/python-test-refactoring).
+This branch is used as a pointer, and not as a feature branch.
+
+1. Rebase the PR onto a recent, good evaluation of `nixos-unstable`
+2. Create a baseline evaluation by force-pushing this revision of `nixos-unstable` to `python-test-refactoring`.
+3. Note the evaluation number (we'll call it `<previous>`)
+4. Push the PR to `python-test-refactoring` and evaluate the PR on Hydra
+5. Create a comparison URL by navigating to the latest build of the PR and adding to the URL `?compare=<previous>`. This is not necessary for the evaluation that comes right after the baseline.
+
+Review the removed tests and newly failed tests using the constructed URL; otherwise you will accidentally compare iterations of the PR instead of changes to the PR base.
+
+As we currently have some flaky tests, newly failing tests are expected, but should be reviewed to make sure that
+ - The number of failures did not increase significantly.
+ - All failures that do occur can reasonably be assumed to fail for a different reason than the changes.
diff --git a/nixos/doc/manual/development/development.md b/nixos/doc/manual/development/development.md
index 6a0dd091b129..76f405c3b29c 100644
--- a/nixos/doc/manual/development/development.md
+++ b/nixos/doc/manual/development/development.md
@@ -10,5 +10,6 @@ bootspec.chapter.md
 what-happens-during-a-system-switch.chapter.md
 writing-documentation.chapter.md
 nixos-tests.chapter.md
+developing-the-test-driver.chapter.md
 testing-installer.chapter.md
 ```
diff --git a/nixos/lib/testing/meta.nix b/nixos/lib/testing/meta.nix
index 65754fe3c541..805b7520edff 100644
--- a/nixos/lib/testing/meta.nix
+++ b/nixos/lib/testing/meta.nix
@@ -22,7 +22,7 @@ in
           };
           timeout = lib.mkOption {
             type = types.nullOr types.int;
-            default = null; # NOTE: null values are filtered out by `meta`.
+            default = 3600;  # 1 hour
             description = mdDoc ''
               The [{option}`test`](#test-opt-test)'s [`meta.timeout`](https://nixos.org/manual/nixpkgs/stable/#var-meta-timeout) in seconds.
             '';
diff --git a/nixos/modules/services/hardware/kanata.nix b/nixos/modules/services/hardware/kanata.nix
index 84265eb8f947..bb730037277b 100644
--- a/nixos/modules/services/hardware/kanata.nix
+++ b/nixos/modules/services/hardware/kanata.nix
@@ -8,19 +8,9 @@ let
   keyboard = {
     options = {
       devices = mkOption {
-        type = types.addCheck (types.listOf types.str)
-          (devices: (length devices) > 0);
+        type = types.listOf types.str;
         example = [ "/dev/input/by-id/usb-0000_0000-event-kbd" ];
-        # TODO replace note with tip, which has not been implemented yet in
-        # nixos/lib/make-options-doc/mergeJSON.py
-        description = mdDoc ''
-          Paths to keyboard devices.
-
-          ::: {.note}
-          To avoid unnecessary triggers of the service unit, unplug devices in
-          the order of the list.
-          :::
-        '';
+        description = mdDoc "Paths to keyboard devices.";
       };
       config = mkOption {
         type = types.lines;
@@ -44,8 +34,10 @@ let
             cap (tap-hold 100 100 caps lctl))
         '';
         description = mdDoc ''
-          Configuration other than `defcfg`. See [example config
-          files](https://github.com/jtroo/kanata) for more information.
+          Configuration other than `defcfg`.
+
+          See [example config files](https://github.com/jtroo/kanata)
+          for more information.
         '';
       };
       extraDefCfg = mkOption {
@@ -53,8 +45,12 @@ let
         default = "";
         example = "danger-enable-cmd yes";
         description = mdDoc ''
-          Configuration of `defcfg` other than `linux-dev`. See [example
-          config files](https://github.com/jtroo/kanata) for more information.
+          Configuration of `defcfg` other than `linux-dev` (generated
+          from the devices option) and
+          `linux-continue-if-no-devs-found` (hardcoded to be yes).
+
+          See [example config files](https://github.com/jtroo/kanata)
+          for more information.
         '';
       };
       extraArgs = mkOption {
@@ -67,8 +63,7 @@ let
         default = null;
         example = 6666;
         description = mdDoc ''
-          Port to run the notification server on. `null` will not run the
-          server.
+          Port to run the TCP server on. `null` will not run the server.
         '';
       };
     };
@@ -76,28 +71,23 @@ let
 
   mkName = name: "kanata-${name}";
 
-  mkDevices = devices: concatStringsSep ":" devices;
+  mkDevices = devices:
+    optionalString ((length devices) > 0) "linux-dev ${concatStringsSep ":" devices}";
 
   mkConfig = name: keyboard: pkgs.writeText "${mkName name}-config.kdb" ''
     (defcfg
       ${keyboard.extraDefCfg}
-      linux-dev ${mkDevices keyboard.devices})
+      ${mkDevices keyboard.devices}
+      linux-continue-if-no-devs-found yes)
 
     ${keyboard.config}
   '';
 
   mkService = name: keyboard: nameValuePair (mkName name) {
-    description = "kanata for ${mkDevices keyboard.devices}";
-
-    # Because path units are used to activate service units, which
-    # will start the old stopped services during "nixos-rebuild
-    # switch", stopIfChanged here is a workaround to make sure new
-    # services are running after "nixos-rebuild switch".
-    stopIfChanged = false;
-
+    wantedBy = [ "multi-user.target" ];
     serviceConfig = {
       ExecStart = ''
-        ${cfg.package}/bin/kanata \
+        ${getExe cfg.package} \
           --cfg ${mkConfig name keyboard} \
           --symlink-path ''${RUNTIME_DIRECTORY}/${name} \
           ${optionalString (keyboard.port != null) "--port ${toString keyboard.port}"} \
@@ -146,37 +136,10 @@ let
       UMask = "0077";
     };
   };
-
-  mkPathName = i: name: "${mkName name}-${toString i}";
-
-  mkPath = name: n: i: device:
-    nameValuePair (mkPathName i name) {
-      description =
-        "${toString (i+1)}/${toString n} kanata trigger for ${name}, watching ${device}";
-      wantedBy = optional (i == 0) "multi-user.target";
-      pathConfig = {
-        PathExists = device;
-        # (ab)use systemd.path to construct a trigger chain so that the
-        # service unit is only started when all paths exist
-        # however, manual of systemd.path says Unit's suffix is not ".path"
-        Unit =
-          if (i + 1) == n
-          then "${mkName name}.service"
-          else "${mkPathName (i + 1) name}.path";
-      };
-      unitConfig.StopPropagatedFrom = optional (i > 0) "${mkName name}.service";
-    };
-
-  mkPaths = name: keyboard:
-    let
-      n = length keyboard.devices;
-    in
-    imap0 (mkPath name n) keyboard.devices
-  ;
 in
 {
   options.services.kanata = {
-    enable = mkEnableOption (lib.mdDoc "kanata");
+    enable = mkEnableOption (mdDoc "kanata");
     package = mkOption {
       type = types.package;
       default = pkgs.kanata;
@@ -201,14 +164,7 @@ in
   config = mkIf cfg.enable {
     hardware.uinput.enable = true;
 
-    systemd = {
-      paths = trivial.pipe cfg.keyboards [
-        (mapAttrsToList mkPaths)
-        concatLists
-        listToAttrs
-      ];
-      services = mapAttrs' mkService cfg.keyboards;
-    };
+    systemd.services = mapAttrs' mkService cfg.keyboards;
   };
 
   meta.maintainers = with maintainers; [ linj ];
diff --git a/nixos/modules/services/mail/maddy.nix b/nixos/modules/services/mail/maddy.nix
index eeb113e204c6..5f3a9b56292d 100644
--- a/nixos/modules/services/mail/maddy.nix
+++ b/nixos/modules/services/mail/maddy.nix
@@ -223,22 +223,59 @@ in {
         '';
       };
 
+      ensureAccounts = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        description = lib.mdDoc ''
+          List of IMAP accounts which get automatically created. Note that for
+          a complete setup, user credentials for these accounts are required too
+          and can be created using the command `maddyctl creds`.
+          This option does not delete accounts which are not (anymore) listed.
+        '';
+        example = [
+          "user1@localhost"
+          "user2@localhost"
+        ];
+      };
+
     };
   };
 
   config = mkIf cfg.enable {
 
     systemd = {
+
       packages = [ pkgs.maddy ];
-      services.maddy = {
-        serviceConfig = {
-          User = cfg.user;
-          Group = cfg.group;
-          StateDirectory = [ "maddy" ];
+      services = {
+        maddy = {
+          serviceConfig = {
+            User = cfg.user;
+            Group = cfg.group;
+            StateDirectory = [ "maddy" ];
+          };
+          restartTriggers = [ config.environment.etc."maddy/maddy.conf".source ];
+          wantedBy = [ "multi-user.target" ];
+        };
+        maddy-ensure-accounts = {
+          script = ''
+            ${optionalString (cfg.ensureAccounts != []) ''
+              ${concatMapStrings (account: ''
+                if ! ${pkgs.maddy}/bin/maddyctl imap-acct list | grep "${account}"; then
+                  ${pkgs.maddy}/bin/maddyctl imap-acct create ${account}
+                fi
+              '') cfg.ensureAccounts}
+            ''}
+          '';
+          serviceConfig = {
+            Type = "oneshot";
+            User= "maddy";
+          };
+          after = [ "maddy.service" ];
+          wantedBy = [ "multi-user.target" ];
         };
-        restartTriggers = [ config.environment.etc."maddy/maddy.conf".source ];
-        wantedBy = [ "multi-user.target" ];
+
       };
+
     };
 
     environment.etc."maddy/maddy.conf" = {
diff --git a/nixos/modules/virtualisation/docker.nix b/nixos/modules/virtualisation/docker.nix
index d9bd10ba1fc8..505cb4e32e1e 100644
--- a/nixos/modules/virtualisation/docker.nix
+++ b/nixos/modules/virtualisation/docker.nix
@@ -163,7 +163,7 @@ in
   ###### implementation
 
   config = mkIf cfg.enable (mkMerge [{
-      boot.kernelModules = [ "bridge" "veth" ];
+      boot.kernelModules = [ "bridge" "veth" "br_netfilter" "xt_nat" ];
       boot.kernel.sysctl = {
         "net.ipv4.conf.all.forwarding" = mkOverride 98 true;
         "net.ipv4.conf.default.forwarding" = mkOverride 98 true;
diff --git a/nixos/tests/maddy.nix b/nixos/tests/maddy.nix
index b9d0416482da..800d254f1770 100644
--- a/nixos/tests/maddy.nix
+++ b/nixos/tests/maddy.nix
@@ -9,6 +9,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
         hostname = "server";
         primaryDomain = "server";
         openFirewall = true;
+        ensureAccounts = [ "postmaster@server" ];
       };
     };
 
@@ -50,7 +51,6 @@ import ./make-test-python.nix ({ pkgs, ... }: {
     server.wait_for_open_port(587)
 
     server.succeed("maddyctl creds create --password test postmaster@server")
-    server.succeed("maddyctl imap-acct create postmaster@server")
 
     client.succeed("send-testmail")
     client.succeed("test-imap")
diff --git a/nixos/tests/pass-secret-service.nix b/nixos/tests/pass-secret-service.nix
index a85a508bfe16..e0dddf0ad29e 100644
--- a/nixos/tests/pass-secret-service.nix
+++ b/nixos/tests/pass-secret-service.nix
@@ -1,6 +1,6 @@
 import ./make-test-python.nix ({ pkgs, lib, ... }: {
   name = "pass-secret-service";
-  meta.maintainers = with lib; [ aidalgol ];
+  meta.maintainers = [ lib.maintainers.aidalgol ];
 
   nodes.machine = { nodes, pkgs, ... }:
     {
diff --git a/nixos/tests/quake3.nix b/nixos/tests/quake3.nix
index 82af1af463d0..ef5fcc41476b 100644
--- a/nixos/tests/quake3.nix
+++ b/nixos/tests/quake3.nix
@@ -1,4 +1,4 @@
-import ./make-test-python.nix ({ pkgs, ...} :
+import ./make-test-python.nix ({ pkgs, lib, ...} :
 
 let
 
@@ -11,9 +11,9 @@ let
     };
 
   # Only allow the demo data to be used (only if it's unfreeRedistributable).
-  unfreePredicate = pkg: with pkgs.lib; let
+  unfreePredicate = pkg: with lib; let
     allowPackageNames = [ "quake3-demodata" "quake3-pointrelease" ];
-    allowLicenses = [ pkgs.lib.licenses.unfreeRedistributable ];
+    allowLicenses = [ lib.licenses.unfreeRedistributable ];
   in elem pkg.pname allowPackageNames &&
      elem (pkg.meta.license or null) allowLicenses;
 
@@ -31,7 +31,7 @@ in
 
 rec {
   name = "quake3";
-  meta = with pkgs.stdenv.lib.maintainers; {
+  meta = with lib.maintainers; {
     maintainers = [ domenkozar eelco ];
   };